2024-11-20 19:40:41,618 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-20 19:40:41,632 main DEBUG Took 0.011960 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-20 19:40:41,633 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-20 19:40:41,633 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-20 19:40:41,634 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-20 19:40:41,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,644 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-20 19:40:41,657 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,660 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,661 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,662 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,663 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,663 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,665 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,666 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,667 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,667 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,669 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,670 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 19:40:41,671 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,672 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-20 19:40:41,674 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 19:40:41,675 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-20 19:40:41,677 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-20 19:40:41,678 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-20 19:40:41,680 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-20 19:40:41,680 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-20 19:40:41,690 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-20 19:40:41,693 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-20 19:40:41,695 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-20 19:40:41,695 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-20 19:40:41,696 main DEBUG createAppenders(={Console})
2024-11-20 19:40:41,697 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-20 19:40:41,697 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-20 19:40:41,698 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-20 19:40:41,698 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-20 19:40:41,699 main DEBUG OutputStream closed
2024-11-20 19:40:41,699 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-20 19:40:41,700 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-20 19:40:41,700 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-20 19:40:41,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-20 19:40:41,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-20 19:40:41,772 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-20 19:40:41,773 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-20 19:40:41,773 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-20 19:40:41,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-20 19:40:41,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-20 19:40:41,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-20 19:40:41,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-20 19:40:41,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-20 19:40:41,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-20 19:40:41,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-20 19:40:41,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-20 19:40:41,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-20 19:40:41,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-20 19:40:41,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-20 19:40:41,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-20 19:40:41,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-20 19:40:41,780 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-20 19:40:41,780 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-20 19:40:41,780 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-20 19:40:41,781 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-20T19:40:42,035 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2
2024-11-20 19:40:42,038 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-20 19:40:42,038 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-20T19:40:42,046 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-20T19:40:42,085 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=11, AvailableMemoryMB=8405
2024-11-20T19:40:42,087 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:40:42,108 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0, deleteOnExit=true
2024-11-20T19:40:42,109 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:40:42,110 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/test.cache.data in system properties and HBase conf
2024-11-20T19:40:42,111 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:40:42,112 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:40:42,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:40:42,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:40:42,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:40:42,204 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-20T19:40:42,290 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:40:42,295 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:40:42,295 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:40:42,296 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:40:42,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:40:42,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:40:42,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:40:42,299 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:40:42,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:40:42,300 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:40:42,301 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:40:42,302 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:40:42,303 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:40:42,303 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:40:42,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:40:42,726 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:40:43,369 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-20T19:40:43,440 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2513ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-20T19:40:43,507 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:40:43,567 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:40:43,586 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:40:43,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:40:43,588 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:40:43,600 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:40:43,602 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:40:43,604 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:40:43,771 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/java.io.tmpdir/jetty-localhost-41029-hadoop-hdfs-3_4_1-tests_jar-_-any-2659894864957587277/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:40:43,777 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:41029}
2024-11-20T19:40:43,777 INFO  [Time-limited test {}] server.Server(415): Started @2851ms
2024-11-20T19:40:43,807 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:40:44,318 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:40:44,325 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:40:44,327 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:40:44,327 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:40:44,327 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:40:44,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:40:44,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:40:44,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/java.io.tmpdir/jetty-localhost-35067-hadoop-hdfs-3_4_1-tests_jar-_-any-12032430943396044057/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:40:44,428 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:35067}
2024-11-20T19:40:44,429 INFO  [Time-limited test {}] server.Server(415): Started @3503ms
2024-11-20T19:40:44,480 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:40:44,586 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:40:44,591 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:40:44,594 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:40:44,595 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:40:44,595 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:40:44,600 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:40:44,601 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:40:44,723 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/java.io.tmpdir/jetty-localhost-35037-hadoop-hdfs-3_4_1-tests_jar-_-any-4355213601923800605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:40:44,724 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:35037}
2024-11-20T19:40:44,724 INFO  [Time-limited test {}] server.Server(415): Started @3799ms
2024-11-20T19:40:44,727 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:40:46,045 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data4/current/BP-1427809238-172.17.0.3-1732131642807/current, will proceed with Du for space computation calculation, 
2024-11-20T19:40:46,045 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data2/current/BP-1427809238-172.17.0.3-1732131642807/current, will proceed with Du for space computation calculation, 
2024-11-20T19:40:46,045 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data3/current/BP-1427809238-172.17.0.3-1732131642807/current, will proceed with Du for space computation calculation, 
2024-11-20T19:40:46,045 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data1/current/BP-1427809238-172.17.0.3-1732131642807/current, will proceed with Du for space computation calculation, 
2024-11-20T19:40:46,080 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:40:46,080 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:40:46,127 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad56ddafffd9295b with lease ID 0xbe3127fecdd24033: Processing first storage report for DS-952efd01-c67c-461e-915d-aaceb7d8eb26 from datanode DatanodeRegistration(127.0.0.1:35233, datanodeUuid=503ae475-45e5-4e34-91ed-55aab59f4075, infoPort=42355, infoSecurePort=0, ipcPort=32775, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807)
2024-11-20T19:40:46,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad56ddafffd9295b with lease ID 0xbe3127fecdd24033: from storage DS-952efd01-c67c-461e-915d-aaceb7d8eb26 node DatanodeRegistration(127.0.0.1:35233, datanodeUuid=503ae475-45e5-4e34-91ed-55aab59f4075, infoPort=42355, infoSecurePort=0, ipcPort=32775, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-20T19:40:46,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe10e426530c0fa8a with lease ID 0xbe3127fecdd24032: Processing first storage report for DS-bb2873bf-b27c-416d-af3a-f7056117e900 from datanode DatanodeRegistration(127.0.0.1:38433, datanodeUuid=b1a0a592-fce8-4cf4-b45c-a2b59b4b2d66, infoPort=44275, infoSecurePort=0, ipcPort=39659, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807)
2024-11-20T19:40:46,129 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe10e426530c0fa8a with lease ID 0xbe3127fecdd24032: from storage DS-bb2873bf-b27c-416d-af3a-f7056117e900 node DatanodeRegistration(127.0.0.1:38433, datanodeUuid=b1a0a592-fce8-4cf4-b45c-a2b59b4b2d66, infoPort=44275, infoSecurePort=0, ipcPort=39659, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:40:46,130 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad56ddafffd9295b with lease ID 0xbe3127fecdd24033: Processing first storage report for DS-af5ea4ae-86f4-4fc3-8863-504ca2a268eb from datanode DatanodeRegistration(127.0.0.1:35233, datanodeUuid=503ae475-45e5-4e34-91ed-55aab59f4075, infoPort=42355, infoSecurePort=0, ipcPort=32775, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807)
2024-11-20T19:40:46,130 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad56ddafffd9295b with lease ID 0xbe3127fecdd24033: from storage DS-af5ea4ae-86f4-4fc3-8863-504ca2a268eb node DatanodeRegistration(127.0.0.1:35233, datanodeUuid=503ae475-45e5-4e34-91ed-55aab59f4075, infoPort=42355, infoSecurePort=0, ipcPort=32775, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:40:46,130 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe10e426530c0fa8a with lease ID 0xbe3127fecdd24032: Processing first storage report for DS-b761d92a-2bae-4403-af41-1ff7ac8845c4 from datanode DatanodeRegistration(127.0.0.1:38433, datanodeUuid=b1a0a592-fce8-4cf4-b45c-a2b59b4b2d66, infoPort=44275, infoSecurePort=0, ipcPort=39659, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807)
2024-11-20T19:40:46,131 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe10e426530c0fa8a with lease ID 0xbe3127fecdd24032: from storage DS-b761d92a-2bae-4403-af41-1ff7ac8845c4 node DatanodeRegistration(127.0.0.1:38433, datanodeUuid=b1a0a592-fce8-4cf4-b45c-a2b59b4b2d66, infoPort=44275, infoSecurePort=0, ipcPort=39659, storageInfo=lv=-57;cid=testClusterID;nsid=895055619;c=1732131642807), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:40:46,149 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2
2024-11-20T19:40:46,215 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/zookeeper_0, clientPort=60731, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:40:46,224 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60731
2024-11-20T19:40:46,237 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:46,240 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:46,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:40:46,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:40:46,860 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a with version=8
2024-11-20T19:40:46,860 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:40:46,934 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-20T19:40:47,192 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:40:47,201 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,201 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,206 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:40:47,206 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,206 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:40:47,325 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:40:47,376 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-20T19:40:47,385 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-20T19:40:47,388 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:40:47,410 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 97631 (auto-detected)
2024-11-20T19:40:47,410 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-20T19:40:47,428 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43333
2024-11-20T19:40:47,447 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43333 connecting to ZooKeeper ensemble=127.0.0.1:60731
2024-11-20T19:40:47,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433330x0, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:40:47,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43333-0x10159c366a10000 connected
2024-11-20T19:40:47,696 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:47,701 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:47,713 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:40:47,717 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a, hbase.cluster.distributed=false
2024-11-20T19:40:47,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:40:47,747 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43333
2024-11-20T19:40:47,748 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43333
2024-11-20T19:40:47,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43333
2024-11-20T19:40:47,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43333
2024-11-20T19:40:47,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43333
2024-11-20T19:40:47,845 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:40:47,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,847 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:40:47,847 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:40:47,847 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:40:47,850 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:40:47,852 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:40:47,853 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46533
2024-11-20T19:40:47,855 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46533 connecting to ZooKeeper ensemble=127.0.0.1:60731
2024-11-20T19:40:47,856 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:47,861 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:47,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465330x0, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:40:47,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465330x0, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:40:47,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46533-0x10159c366a10001 connected
2024-11-20T19:40:47,898 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:40:47,906 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:40:47,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:40:47,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:40:47,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46533
2024-11-20T19:40:47,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46533
2024-11-20T19:40:47,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46533
2024-11-20T19:40:47,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46533
2024-11-20T19:40:47,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46533
2024-11-20T19:40:47,934 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:43333
2024-11-20T19:40:47,935 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,43333,1732131647044
2024-11-20T19:40:47,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:40:47,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:40:47,948 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,43333,1732131647044
2024-11-20T19:40:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:40:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:47,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:47,978 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:40:47,979 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,43333,1732131647044 from backup master directory
2024-11-20T19:40:47,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:40:47,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,43333,1732131647044
2024-11-20T19:40:47,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:40:47,988 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:40:47,988 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,43333,1732131647044
2024-11-20T19:40:47,990 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-20T19:40:47,992 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-20T19:40:48,044 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase.id] with ID: 6a8997cf-11fa-463f-8918-8d15e215d3e4
2024-11-20T19:40:48,044 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/.tmp/hbase.id
2024-11-20T19:40:48,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:40:48,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:40:48,058 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/.tmp/hbase.id]:[hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase.id]
2024-11-20T19:40:48,102 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:48,107 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:40:48,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms.
2024-11-20T19:40:48,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:40:48,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:40:48,169 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:40:48,170 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:40:48,175 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:40:48,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:40:48,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:40:48,220 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store
2024-11-20T19:40:48,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:40:48,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:40:48,244 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-20T19:40:48,248 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:48,249 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:40:48,249 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:40:48,249 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:40:48,251 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms
2024-11-20T19:40:48,251 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:40:48,251 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:40:48,252 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131648249Disabling compacts and flushes for region at 1732131648249Disabling writes for close at 1732131648251 (+2 ms)Writing region close event to WAL at 1732131648251Closed at 1732131648251
2024-11-20T19:40:48,255 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/.initializing
2024-11-20T19:40:48,255 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/WALs/e14ef07817f4,43333,1732131647044
2024-11-20T19:40:48,276 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C43333%2C1732131647044, suffix=, logDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/WALs/e14ef07817f4,43333,1732131647044, archiveDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/oldWALs, maxLogs=10
2024-11-20T19:40:48,284 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C43333%2C1732131647044.1732131648280
2024-11-20T19:40:48,302 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/WALs/e14ef07817f4,43333,1732131647044/e14ef07817f4%2C43333%2C1732131647044.1732131648280
2024-11-20T19:40:48,310 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:40:48,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:40:48,314 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:48,317 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,318 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,352 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:40:48,375 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:40:48,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,381 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:48,382 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:40:48,385 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:40:48,386 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,387 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:40:48,387 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:40:48,390 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:40:48,390 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:40:48,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:40:48,394 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:40:48,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,395 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:40:48,396 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,400 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,401 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,407 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,408 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,411 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:40:48,415 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:40:48,419 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:40:48,420 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756675, jitterRate=-0.03783811628818512}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:40:48,428 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131648329Initializing all the Stores at 1732131648331 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131648332 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131648332Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131648333 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131648333Cleaning up temporary data from old regions at 1732131648408 (+75 ms)Region opened successfully at 1732131648427 (+19 ms)
2024-11-20T19:40:48,429 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:40:48,461 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2800d548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:40:48,487 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:40:48,496 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:40:48,497 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:40:48,499 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:40:48,500 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-20T19:40:48,504 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-20T19:40:48,505 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:40:48,528 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:40:48,535 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:40:48,587 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:40:48,591 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:40:48,594 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:40:48,607 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:40:48,610 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:40:48,614 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:40:48,618 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:40:48,620 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:40:48,629 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:40:48,646 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:40:48,659 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:40:48,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:40:48,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:40:48,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,676 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,43333,1732131647044, sessionid=0x10159c366a10000, setting cluster-up flag (Was=false)
2024-11-20T19:40:48,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,734 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:40:48,737 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,43333,1732131647044
2024-11-20T19:40:48,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:48,787 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:40:48,791 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,43333,1732131647044
2024-11-20T19:40:48,800 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:40:48,824 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(746): ClusterId : 6a8997cf-11fa-463f-8918-8d15e215d3e4
2024-11-20T19:40:48,826 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:40:48,842 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:40:48,842 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:40:48,852 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:40:48,852 DEBUG [RS:0;e14ef07817f4:46533 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6454a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:40:48,868 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:46533
2024-11-20T19:40:48,867 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:40:48,870 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:40:48,870 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:40:48,871 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:40:48,873 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,43333,1732131647044 with port=46533, startcode=1732131647812
2024-11-20T19:40:48,876 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:40:48,881 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:40:48,883 DEBUG [RS:0;e14ef07817f4:46533 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:40:48,886 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,43333,1732131647044
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:40:48,892 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:40:48,892 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:40:48,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:48,895 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131678895
2024-11-20T19:40:48,896 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:40:48,897 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:40:48,899 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:40:48,900 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:40:48,900 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:40:48,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:40:48,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:40:48,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:40:48,902 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:48,906 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,907 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:40:48,908 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:40:48,909 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:40:48,910 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:40:48,912 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:40:48,912 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:40:48,914 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131648913,5,FailOnTimeoutGroup]
2024-11-20T19:40:48,918 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131648914,5,FailOnTimeoutGroup]
2024-11-20T19:40:48,918 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:48,919 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:40:48,919 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:48,920 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:48,924 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:40:48,925 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:40:48,927 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:40:48,928 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a
2024-11-20T19:40:48,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:40:48,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:40:48,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:48,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:40:48,947 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:40:48,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:48,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:40:48,952 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:40:48,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,953 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55523, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:40:48,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:48,954 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:40:48,956 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:40:48,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,958 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:48,958 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:40:48,959 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43333 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,46533,1732131647812
2024-11-20T19:40:48,961 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43333 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:48,961 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:40:48,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:48,963 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:48,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:40:48,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740
2024-11-20T19:40:48,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740
2024-11-20T19:40:48,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:40:48,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:40:48,970 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:40:48,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:40:48,977 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a
2024-11-20T19:40:48,977 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:40:48,977 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43327
2024-11-20T19:40:48,977 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:40:48,978 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=797414, jitterRate=0.01396527886390686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:40:48,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131648941Initializing all the Stores at 1732131648943 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131648943Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131648944 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131648944Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131648944Cleaning up temporary data from old regions at 1732131648969 (+25 ms)Region opened successfully at 1732131648982 (+13 ms)
2024-11-20T19:40:48,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:40:48,982 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:40:48,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:40:48,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:40:48,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:40:48,984 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:40:48,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131648982Disabling compacts and flushes for region at 1732131648982Disabling writes for close at 1732131648982Writing region close event to WAL at 1732131648983 (+1 ms)Closed at 1732131648984 (+1 ms)
2024-11-20T19:40:48,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:40:48,987 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:40:48,987 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:40:48,987 DEBUG [RS:0;e14ef07817f4:46533 {}] zookeeper.ZKUtil(111): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,46533,1732131647812
2024-11-20T19:40:48,988 WARN  [RS:0;e14ef07817f4:46533 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:40:48,988 INFO  [RS:0;e14ef07817f4:46533 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:40:48,988 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812
2024-11-20T19:40:48,990 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,46533,1732131647812]
2024-11-20T19:40:48,995 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:40:49,003 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:40:49,006 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:40:49,014 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:40:49,029 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:40:49,033 INFO  [RS:0;e14ef07817f4:46533 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:40:49,033 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,034 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:40:49,038 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:40:49,039 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,040 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:40:49,041 DEBUG [RS:0;e14ef07817f4:46533 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:40:49,042 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,042 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,042 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,043 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,043 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,043 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,46533,1732131647812-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:40:49,060 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:40:49,062 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,46533,1732131647812-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,062 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,062 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.Replication(171): e14ef07817f4,46533,1732131647812 started
2024-11-20T19:40:49,080 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,081 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,46533,1732131647812, RpcServer on e14ef07817f4/172.17.0.3:46533, sessionid=0x10159c366a10001
2024-11-20T19:40:49,082 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:40:49,082 DEBUG [RS:0;e14ef07817f4:46533 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,46533,1732131647812
2024-11-20T19:40:49,082 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,46533,1732131647812'
2024-11-20T19:40:49,082 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:40:49,083 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:40:49,084 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:40:49,084 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:40:49,084 DEBUG [RS:0;e14ef07817f4:46533 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,46533,1732131647812
2024-11-20T19:40:49,084 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,46533,1732131647812'
2024-11-20T19:40:49,084 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:40:49,085 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:40:49,086 DEBUG [RS:0;e14ef07817f4:46533 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:40:49,086 INFO  [RS:0;e14ef07817f4:46533 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:40:49,086 INFO  [RS:0;e14ef07817f4:46533 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:40:49,157 WARN  [e14ef07817f4:43333 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:40:49,198 INFO  [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C46533%2C1732131647812, suffix=, logDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812, archiveDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs, maxLogs=32
2024-11-20T19:40:49,202 INFO  [RS:0;e14ef07817f4:46533 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131649202
2024-11-20T19:40:49,211 INFO  [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131649202
2024-11-20T19:40:49,214 DEBUG [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:40:49,413 DEBUG [e14ef07817f4:43333 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:40:49,427 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:49,434 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,46533,1732131647812, state=OPENING
2024-11-20T19:40:49,492 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:40:49,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:49,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:40:49,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:40:49,506 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:40:49,509 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:40:49,513 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,46533,1732131647812}]
2024-11-20T19:40:49,695 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:40:49,698 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53623, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:40:49,708 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:40:49,708 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:40:49,712 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C46533%2C1732131647812.meta, suffix=.meta, logDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812, archiveDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs, maxLogs=32
2024-11-20T19:40:49,714 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.meta.1732131649714.meta
2024-11-20T19:40:49,722 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.meta.1732131649714.meta
2024-11-20T19:40:49,724 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:40:49,725 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:40:49,727 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:40:49,729 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:40:49,733 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:40:49,737 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:40:49,737 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:49,738 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:40:49,738 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:40:49,741 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:40:49,743 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:40:49,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:49,744 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:49,744 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:40:49,746 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:40:49,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:49,747 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:49,747 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:40:49,748 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:40:49,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:49,749 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:49,749 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:40:49,751 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:40:49,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:49,752 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:40:49,752 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:40:49,754 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740
2024-11-20T19:40:49,757 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740
2024-11-20T19:40:49,759 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:40:49,759 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:40:49,760 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:40:49,764 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:40:49,766 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802858, jitterRate=0.020887449383735657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:40:49,766 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:40:49,768 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131649738Writing region info on filesystem at 1732131649739 (+1 ms)Initializing all the Stores at 1732131649740 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131649740Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131649741 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131649741Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131649741Cleaning up temporary data from old regions at 1732131649760 (+19 ms)Running coprocessor post-open hooks at 1732131649766 (+6 ms)Region opened successfully at 1732131649768 (+2 ms)
2024-11-20T19:40:49,774 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131649688
2024-11-20T19:40:49,784 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:40:49,785 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:40:49,786 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:49,789 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,46533,1732131647812, state=OPEN
2024-11-20T19:40:49,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:40:49,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:40:49,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:40:49,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:40:49,830 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:49,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:40:49,841 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,46533,1732131647812 in 318 msec
2024-11-20T19:40:49,847 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:40:49,848 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 847 msec
2024-11-20T19:40:49,849 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:40:49,849 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:40:49,869 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:40:49,870 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,46533,1732131647812, seqNum=-1]
2024-11-20T19:40:49,888 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:40:49,890 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42821, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:40:49,908 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0800 sec
2024-11-20T19:40:49,909 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131649909, completionTime=-1
2024-11-20T19:40:49,911 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:40:49,911 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:40:49,936 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:40:49,936 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131709936
2024-11-20T19:40:49,936 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732131769936
2024-11-20T19:40:49,936 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec
2024-11-20T19:40:49,939 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,939 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,939 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,941 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:43333, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,941 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,942 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:49,948 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:40:49,969 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.981sec
2024-11-20T19:40:49,970 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:40:49,971 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:40:49,972 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:40:49,973 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:40:49,973 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:40:49,974 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:40:49,974 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:40:49,982 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:40:49,983 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:40:49,983 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43333,1732131647044-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:40:50,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:40:50,053 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-20T19:40:50,054 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-20T19:40:50,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,43333,-1 for getting cluster id
2024-11-20T19:40:50,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:40:50,075 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6a8997cf-11fa-463f-8918-8d15e215d3e4'
2024-11-20T19:40:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:40:50,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6a8997cf-11fa-463f-8918-8d15e215d3e4"

2024-11-20T19:40:50,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120d12d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:40:50,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,43333,-1]
2024-11-20T19:40:50,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:40:50,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:40:50,088 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:40:50,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:40:50,092 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:40:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,46533,1732131647812, seqNum=-1]
2024-11-20T19:40:50,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:40:50,102 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:40:50,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,43333,1732131647044
2024-11-20T19:40:50,120 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:40:50,126 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:40:50,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T19:40:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e14ef07817f4,43333,1732131647044
2024-11-20T19:40:50,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bfb5e00
2024-11-20T19:40:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T19:40:50,138 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T19:40:50,140 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T19:40:50,140 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T19:40:50,143 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:40:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-20T19:40:50,153 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T19:40:50,155 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-20T19:40:50,155 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:50,158 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T19:40:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:40:50,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741835_1011 (size=389)
2024-11-20T19:40:50,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741835_1011 (size=389)
2024-11-20T19:40:50,218 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c1ce36384d21b3e1686732853b6417d5, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a
2024-11-20T19:40:50,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741836_1012 (size=72)
2024-11-20T19:40:50,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741836_1012 (size=72)
2024-11-20T19:40:50,228 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:50,228 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing c1ce36384d21b3e1686732853b6417d5, disabling compactions & flushes
2024-11-20T19:40:50,229 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,229 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,229 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5. after waiting 0 ms
2024-11-20T19:40:50,229 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,229 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,229 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for c1ce36384d21b3e1686732853b6417d5:
Waiting for close lock at 1732131650228Disabling compacts and flushes for region at 1732131650228Disabling writes for close at 1732131650229 (+1 ms)Writing region close event to WAL at 1732131650229Closed at 1732131650229
2024-11-20T19:40:50,231 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T19:40:50,235 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732131650231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131650231"}]},"ts":"1732131650231"}
2024-11-20T19:40:50,240 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T19:40:50,242 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T19:40:50,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131650242"}]},"ts":"1732131650242"}
2024-11-20T19:40:50,248 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-20T19:40:50,250 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c1ce36384d21b3e1686732853b6417d5, ASSIGN}]
2024-11-20T19:40:50,253 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c1ce36384d21b3e1686732853b6417d5, ASSIGN
2024-11-20T19:40:50,255 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c1ce36384d21b3e1686732853b6417d5, ASSIGN; state=OFFLINE, location=e14ef07817f4,46533,1732131647812; forceNewPlan=false, retain=false
2024-11-20T19:40:50,406 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c1ce36384d21b3e1686732853b6417d5, regionState=OPENING, regionLocation=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:50,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c1ce36384d21b3e1686732853b6417d5, ASSIGN because future has completed
2024-11-20T19:40:50,415 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1ce36384d21b3e1686732853b6417d5, server=e14ef07817f4,46533,1732131647812}]
2024-11-20T19:40:50,582 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,583 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c1ce36384d21b3e1686732853b6417d5, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:40:50,584 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,584 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:40:50,584 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,585 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,588 INFO  [StoreOpener-c1ce36384d21b3e1686732853b6417d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c1ce36384d21b3e1686732853b6417d5 
2024-11-20T19:40:50,591 INFO  [StoreOpener-c1ce36384d21b3e1686732853b6417d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1ce36384d21b3e1686732853b6417d5 columnFamilyName info
2024-11-20T19:40:50,591 DEBUG [StoreOpener-c1ce36384d21b3e1686732853b6417d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:40:50,592 INFO  [StoreOpener-c1ce36384d21b3e1686732853b6417d5-1 {}] regionserver.HStore(327): Store=c1ce36384d21b3e1686732853b6417d5/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:40:50,592 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,594 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,594 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,595 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,595 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,598 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,601 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:40:50,602 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c1ce36384d21b3e1686732853b6417d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698513, jitterRate=-0.11179482936859131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:40:50,602 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:40:50,603 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c1ce36384d21b3e1686732853b6417d5:
Running coprocessor pre-open hook at 1732131650585Writing region info on filesystem at 1732131650585Initializing all the Stores at 1732131650588 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131650588Cleaning up temporary data from old regions at 1732131650595 (+7 ms)Running coprocessor post-open hooks at 1732131650602 (+7 ms)Region opened successfully at 1732131650603 (+1 ms)
2024-11-20T19:40:50,605 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5., pid=6, masterSystemTime=1732131650570
2024-11-20T19:40:50,609 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,609 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:40:50,610 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c1ce36384d21b3e1686732853b6417d5, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,46533,1732131647812
2024-11-20T19:40:50,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1ce36384d21b3e1686732853b6417d5, server=e14ef07817f4,46533,1732131647812 because future has completed
2024-11-20T19:40:50,620 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T19:40:50,620 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c1ce36384d21b3e1686732853b6417d5, server=e14ef07817f4,46533,1732131647812 in 201 msec
2024-11-20T19:40:50,624 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T19:40:50,624 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=c1ce36384d21b3e1686732853b6417d5, ASSIGN in 370 msec
2024-11-20T19:40:50,626 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T19:40:50,626 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131650626"}]},"ts":"1732131650626"}
2024-11-20T19:40:50,629 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-20T19:40:50,631 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T19:40:50,634 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 485 msec
2024-11-20T19:40:55,251 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-20T19:40:55,298 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:40:55,299 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-20T19:40:57,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:40:57,375 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T19:40:57,378 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-20T19:40:57,378 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T19:40:57,380 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:40:57,380 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T19:40:57,381 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T19:40:57,381 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-20T19:41:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43333 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:41:00,211 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-20T19:41:00,219 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-20T19:41:00,226 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-20T19:41:00,226 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:41:00,227 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131660227
2024-11-20T19:41:00,236 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:00,236 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:00,236 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:00,236 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:00,237 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:00,237 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131649202 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131660227
2024-11-20T19:41:00,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42355:42355),(127.0.0.1/127.0.0.1:44275:44275)]
2024-11-20T19:41:00,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131649202 is not closed yet, will try archiving it next time
2024-11-20T19:41:00,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741833_1009 (size=451)
2024-11-20T19:41:00,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741833_1009 (size=451)
2024-11-20T19:41:00,242 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131649202 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131649202
2024-11-20T19:41:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5., hostname=e14ef07817f4,46533,1732131647812, seqNum=2]
2024-11-20T19:41:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:41:12,301 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce36384d21b3e1686732853b6417d5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:41:12,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/6dbfccdf4f804549aeaa2f73bd2f7a9d is 1080, key is row0001/info:/1732131660250/Put/seqid=0
2024-11-20T19:41:12,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741838_1014 (size=12509)
2024-11-20T19:41:12,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741838_1014 (size=12509)
2024-11-20T19:41:12,370 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/6dbfccdf4f804549aeaa2f73bd2f7a9d
2024-11-20T19:41:12,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/6dbfccdf4f804549aeaa2f73bd2f7a9d as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d
2024-11-20T19:41:12,429 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d, entries=7, sequenceid=11, filesize=12.2 K
2024-11-20T19:41:12,437 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 138ms, sequenceid=11, compaction requested=false
2024-11-20T19:41:12,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce36384d21b3e1686732853b6417d5:

2024-11-20T19:41:16,147 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:41:20,323 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131680322
2024-11-20T19:41:20,536 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:20,537 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:20,537 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:20,537 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:20,537 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:20,537 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:20,538 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131660227 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131680322
2024-11-20T19:41:20,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42355:42355),(127.0.0.1/127.0.0.1:44275:44275)]
2024-11-20T19:41:20,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131660227 is not closed yet, will try archiving it next time
2024-11-20T19:41:20,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741837_1013 (size=12399)
2024-11-20T19:41:20,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741837_1013 (size=12399)
2024-11-20T19:41:20,744 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:22,949 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:25,156 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:27,366 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:41:27,368 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce36384d21b3e1686732853b6417d5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:41:27,572 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:27,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/3d4fee02231544019b27f2d6f3f59d03 is 1080, key is row0008/info:/1732131674299/Put/seqid=0
2024-11-20T19:41:27,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741840_1016 (size=12509)
2024-11-20T19:41:27,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741840_1016 (size=12509)
2024-11-20T19:41:27,594 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/3d4fee02231544019b27f2d6f3f59d03
2024-11-20T19:41:27,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/3d4fee02231544019b27f2d6f3f59d03 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03
2024-11-20T19:41:27,615 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03, entries=7, sequenceid=21, filesize=12.2 K
2024-11-20T19:41:27,817 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:27,817 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 450ms, sequenceid=21, compaction requested=false
2024-11-20T19:41:27,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce36384d21b3e1686732853b6417d5:

2024-11-20T19:41:27,817 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-20T19:41:27,817 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:41:27,818 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d because midkey is the same as first or last row
2024-11-20T19:41:29,574 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:30,037 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T19:41:30,037 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T19:41:31,783 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:31,790 WARN  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:31,792 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C46533%2C1732131647812:(num 1732131680322) roll requested
2024-11-20T19:41:31,793 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131691792
2024-11-20T19:41:32,006 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 211 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:32,007 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:32,007 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:32,007 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:32,008 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:32,008 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:32,009 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131680322 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131691792
2024-11-20T19:41:32,010 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42355:42355),(127.0.0.1/127.0.0.1:44275:44275)]
2024-11-20T19:41:32,011 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131680322 is not closed yet, will try archiving it next time
2024-11-20T19:41:32,011 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131660227 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131660227
2024-11-20T19:41:32,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741839_1015 (size=7739)
2024-11-20T19:41:32,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741839_1015 (size=7739)
2024-11-20T19:41:33,991 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:35,584 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region c1ce36384d21b3e1686732853b6417d5, had cached 0 bytes from a total of 25018
2024-11-20T19:41:36,199 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:38,413 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:40,622 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 203 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:42,625 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T19:41:42,625 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131702625
2024-11-20T19:41:46,148 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:41:47,685 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5057 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:47,688 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5057 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:41:47,688 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C46533%2C1732131647812:(num 1732131702625) roll requested
2024-11-20T19:41:47,688 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:47,689 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:47,689 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:47,689 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:47,689 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:47,689 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131691792 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131702625
2024-11-20T19:41:47,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:41:47,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131691792 is not closed yet, will try archiving it next time
2024-11-20T19:41:47,691 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131707690
2024-11-20T19:41:47,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741841_1017 (size=4753)
2024-11-20T19:41:47,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741841_1017 (size=4753)
2024-11-20T19:41:52,696 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5003 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:52,696 WARN  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5003 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:41:52,697 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce36384d21b3e1686732853b6417d5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:41:52,763 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5068 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:52,763 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5068 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:54,699 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T19:41:57,759 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5058 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:57,759 WARN  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5058 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:41:57,759 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:57,759 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:57,759 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:57,759 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:57,760 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:41:57,760 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131702625 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131707690
2024-11-20T19:41:57,763 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42355:42355),(127.0.0.1/127.0.0.1:44275:44275)]
2024-11-20T19:41:57,763 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131702625 is not closed yet, will try archiving it next time
2024-11-20T19:41:57,763 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C46533%2C1732131647812:(num 1732131707690) roll requested
2024-11-20T19:41:57,764 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131717764
2024-11-20T19:41:57,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741842_1018 (size=1569)
2024-11-20T19:41:57,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741842_1018 (size=1569)
2024-11-20T19:41:57,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/082903f67e714960b7850d843f660cb1 is 1080, key is row0015/info:/1732131689371/Put/seqid=0
2024-11-20T19:41:57,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741844_1020 (size=12509)
2024-11-20T19:41:57,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741844_1020 (size=12509)
2024-11-20T19:41:57,777 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/082903f67e714960b7850d843f660cb1
2024-11-20T19:41:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/082903f67e714960b7850d843f660cb1 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1
2024-11-20T19:41:57,795 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1, entries=7, sequenceid=31, filesize=12.2 K
2024-11-20T19:42:02,780 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:42:02,781 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:42:02,797 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:42:02,797 WARN  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK], DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK]]
2024-11-20T19:42:02,797 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 10100ms, sequenceid=31, compaction requested=true
2024-11-20T19:42:02,797 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce36384d21b3e1686732853b6417d5:

2024-11-20T19:42:02,797 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,797 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-20T19:42:02,797 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,797 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:02,797 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,797 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d because midkey is the same as first or last row
2024-11-20T19:42:02,798 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,798 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131707690 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131717764
2024-11-20T19:42:02,799 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:42:02,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c1ce36384d21b3e1686732853b6417d5:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:42:02,799 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131707690 is not closed yet, will try archiving it next time
2024-11-20T19:42:02,799 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131680322 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131680322
2024-11-20T19:42:02,799 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C46533%2C1732131647812:(num 1732131722799) roll requested
2024-11-20T19:42:02,800 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131722799
2024-11-20T19:42:02,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741843_1019 (size=438)
2024-11-20T19:42:02,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741843_1019 (size=438)
2024-11-20T19:42:02,802 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131691792 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131691792
2024-11-20T19:42:02,803 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:42:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:02,804 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131702625 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131702625
2024-11-20T19:42:02,805 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131707690 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131707690
2024-11-20T19:42:02,806 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:42:02,808 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HStore(1541): c1ce36384d21b3e1686732853b6417d5/info is initiating minor compaction (all files)
2024-11-20T19:42:02,808 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,808 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,808 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,808 INFO  [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c1ce36384d21b3e1686732853b6417d5/info in TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:02,808 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,808 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,809 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131717764 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131722799
2024-11-20T19:42:02,809 INFO  [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1] into tmpdir=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp, totalSize=36.6 K
2024-11-20T19:42:02,810 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dbfccdf4f804549aeaa2f73bd2f7a9d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732131660250
2024-11-20T19:42:02,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741845_1021 (size=93)
2024-11-20T19:42:02,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741845_1021 (size=93)
2024-11-20T19:42:02,811 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4fee02231544019b27f2d6f3f59d03, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732131674299
2024-11-20T19:42:02,812 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131717764 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs/e14ef07817f4%2C46533%2C1732131647812.1732131717764
2024-11-20T19:42:02,812 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] compactions.Compactor(225): Compacting 082903f67e714960b7850d843f660cb1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732131689371
2024-11-20T19:42:02,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:42:02,818 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46533%2C1732131647812.1732131722818
2024-11-20T19:42:02,825 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,825 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,825 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,825 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,826 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:02,826 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131722799 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/WALs/e14ef07817f4,46533,1732131647812/e14ef07817f4%2C46533%2C1732131647812.1732131722818
2024-11-20T19:42:02,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741846_1022 (size=1258)
2024-11-20T19:42:02,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741846_1022 (size=1258)
2024-11-20T19:42:02,836 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44275:44275),(127.0.0.1/127.0.0.1:42355:42355)]
2024-11-20T19:42:02,847 INFO  [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c1ce36384d21b3e1686732853b6417d5#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:42:02,848 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/80eab6a56bf5405abc2e6649abea6f46 is 1080, key is row0001/info:/1732131660250/Put/seqid=0
2024-11-20T19:42:02,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741848_1024 (size=27710)
2024-11-20T19:42:02,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741848_1024 (size=27710)
2024-11-20T19:42:02,867 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/80eab6a56bf5405abc2e6649abea6f46 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/80eab6a56bf5405abc2e6649abea6f46
2024-11-20T19:42:02,886 INFO  [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c1ce36384d21b3e1686732853b6417d5/info of c1ce36384d21b3e1686732853b6417d5 into 80eab6a56bf5405abc2e6649abea6f46(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:42:02,886 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c1ce36384d21b3e1686732853b6417d5:

2024-11-20T19:42:02,888 INFO  [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5., storeName=c1ce36384d21b3e1686732853b6417d5/info, priority=13, startTime=1732131722799; duration=0sec
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/80eab6a56bf5405abc2e6649abea6f46 because midkey is the same as first or last row
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/80eab6a56bf5405abc2e6649abea6f46 because midkey is the same as first or last row
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:02,888 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/80eab6a56bf5405abc2e6649abea6f46 because midkey is the same as first or last row
2024-11-20T19:42:02,889 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:02,889 DEBUG [RS:0;e14ef07817f4:46533-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c1ce36384d21b3e1686732853b6417d5:info
2024-11-20T19:42:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] regionserver.HRegion(8855): Flush requested on c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:42:14,870 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c1ce36384d21b3e1686732853b6417d5 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:42:14,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/c6ee6dbaccee47709490a2078fae784d is 1080, key is row0022/info:/1732131722820/Put/seqid=0
2024-11-20T19:42:14,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741849_1025 (size=12509)
2024-11-20T19:42:14,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741849_1025 (size=12509)
2024-11-20T19:42:14,886 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/c6ee6dbaccee47709490a2078fae784d
2024-11-20T19:42:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/c6ee6dbaccee47709490a2078fae784d as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/c6ee6dbaccee47709490a2078fae784d
2024-11-20T19:42:14,905 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/c6ee6dbaccee47709490a2078fae784d, entries=7, sequenceid=42, filesize=12.2 K
2024-11-20T19:42:14,907 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 38ms, sequenceid=42, compaction requested=false
2024-11-20T19:42:14,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c1ce36384d21b3e1686732853b6417d5:

2024-11-20T19:42:14,907 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-20T19:42:14,907 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:14,907 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/80eab6a56bf5405abc2e6649abea6f46 because midkey is the same as first or last row
2024-11-20T19:42:16,148 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:42:20,585 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region c1ce36384d21b3e1686732853b6417d5, had cached 0 bytes from a total of 40219
2024-11-20T19:42:21,006 INFO  [FSHLog-0-hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a-prefix:e14ef07817f4,46533,1732131647812 {}] wal.AbstractFSWAL(1368): Slow sync cost: 113 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38433,DS-bb2873bf-b27c-416d-af3a-f7056117e900,DISK], DatanodeInfoWithStorage[127.0.0.1:35233,DS-952efd01-c67c-461e-915d-aaceb7d8eb26,DISK]]
2024-11-20T19:42:23,007 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:42:23,009 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:42:23,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:23,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:23,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:23,019 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:42:23,020 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:42:23,020 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=887671035, stopped=false
2024-11-20T19:42:23,020 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,43333,1732131647044
2024-11-20T19:42:23,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:23,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:23,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:23,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:23,097 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:42:23,097 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:42:23,098 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:23,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:23,098 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:23,098 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:23,098 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,46533,1732131647812' *****
2024-11-20T19:42:23,098 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:42:23,099 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:42:23,099 INFO  [RS:0;e14ef07817f4:46533 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:42:23,099 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:42:23,099 INFO  [RS:0;e14ef07817f4:46533 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:42:23,099 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(3091): Received CLOSE for c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:42:23,100 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,46533,1732131647812
2024-11-20T19:42:23,100 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:42:23,100 INFO  [RS:0;e14ef07817f4:46533 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:46533.
2024-11-20T19:42:23,101 DEBUG [RS:0;e14ef07817f4:46533 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:23,101 DEBUG [RS:0;e14ef07817f4:46533 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:23,101 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c1ce36384d21b3e1686732853b6417d5, disabling compactions & flushes
2024-11-20T19:42:23,101 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:23,101 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:42:23,101 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:23,101 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:42:23,101 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:42:23,101 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5. after waiting 0 ms
2024-11-20T19:42:23,101 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:23,101 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:42:23,102 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c1ce36384d21b3e1686732853b6417d5 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-20T19:42:23,102 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T19:42:23,102 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:42:23,102 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1325): Online Regions={c1ce36384d21b3e1686732853b6417d5=TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:42:23,102 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:42:23,102 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:42:23,103 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:42:23,103 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:42:23,103 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:42:23,103 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-20T19:42:23,108 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/691d737264f547e0b54c3ff4af1a7975 is 1080, key is row0029/info:/1732131736874/Put/seqid=0
2024-11-20T19:42:23,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741850_1026 (size=8193)
2024-11-20T19:42:23,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741850_1026 (size=8193)
2024-11-20T19:42:23,129 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/info/8232b9c7bd6f425d88e7658d847e4075 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5./info:regioninfo/1732131650610/Put/seqid=0
2024-11-20T19:42:23,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741851_1027 (size=7016)
2024-11-20T19:42:23,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741851_1027 (size=7016)
2024-11-20T19:42:23,136 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/info/8232b9c7bd6f425d88e7658d847e4075
2024-11-20T19:42:23,160 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/ns/664db17e281f4d0b973f36b2e3e2ed8f is 43, key is default/ns:d/1732131649894/Put/seqid=0
2024-11-20T19:42:23,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741852_1028 (size=5153)
2024-11-20T19:42:23,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741852_1028 (size=5153)
2024-11-20T19:42:23,303 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:42:23,503 DEBUG [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c1ce36384d21b3e1686732853b6417d5
2024-11-20T19:42:23,518 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/691d737264f547e0b54c3ff4af1a7975
2024-11-20T19:42:23,534 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/.tmp/info/691d737264f547e0b54c3ff4af1a7975 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/691d737264f547e0b54c3ff4af1a7975
2024-11-20T19:42:23,542 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/691d737264f547e0b54c3ff4af1a7975, entries=3, sequenceid=48, filesize=8.0 K
2024-11-20T19:42:23,544 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 443ms, sequenceid=48, compaction requested=true
2024-11-20T19:42:23,545 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1] to archive
2024-11-20T19:42:23,548 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:42:23,552 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/6dbfccdf4f804549aeaa2f73bd2f7a9d
2024-11-20T19:42:23,554 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/3d4fee02231544019b27f2d6f3f59d03
2024-11-20T19:42:23,556 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1 to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/info/082903f67e714960b7850d843f660cb1
2024-11-20T19:42:23,568 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/ns/664db17e281f4d0b973f36b2e3e2ed8f
2024-11-20T19:42:23,566 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e14ef07817f4:43333 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-20T19:42:23,571 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [6dbfccdf4f804549aeaa2f73bd2f7a9d=12509, 3d4fee02231544019b27f2d6f3f59d03=12509, 082903f67e714960b7850d843f660cb1=12509]
2024-11-20T19:42:23,576 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/default/TestLogRolling-testSlowSyncLogRolling/c1ce36384d21b3e1686732853b6417d5/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-20T19:42:23,578 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:23,579 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c1ce36384d21b3e1686732853b6417d5:
Waiting for close lock at 1732131743100Running coprocessor pre-close hooks at 1732131743101 (+1 ms)Disabling compacts and flushes for region at 1732131743101Disabling writes for close at 1732131743101Obtaining lock to block concurrent updates at 1732131743102 (+1 ms)Preparing flush snapshotting stores in c1ce36384d21b3e1686732853b6417d5 at 1732131743102Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732131743102Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5. at 1732131743104 (+2 ms)Flushing c1ce36384d21b3e1686732853b6417d5/info: creating writer at 1732131743104Flushing c1ce36384d21b3e1686732853b6417d5/info: appending metadata at 1732131743108 (+4 ms)Flushing c1ce36384d21b3e1686732853b6417d5/info: closing flushed file at 1732131743108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eb6c6e3: reopening flushed file at 1732131743533 (+425 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for c1ce36384d21b3e1686732853b6417d5 in 443ms, sequenceid=48, compaction requested=true at 1732131743544 (+11 ms)Writing region close event to WAL at 1732131743572 (+28 ms)Running coprocessor post-close hooks at 1732131743577 (+5 ms)Closed at 1732131743578 (+1 ms)
2024-11-20T19:42:23,579 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732131650140.c1ce36384d21b3e1686732853b6417d5.
2024-11-20T19:42:23,591 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/table/035943993d33434d9f23a29c0daf42d9 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732131650626/Put/seqid=0
2024-11-20T19:42:23,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741853_1029 (size=5396)
2024-11-20T19:42:23,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741853_1029 (size=5396)
2024-11-20T19:42:23,597 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/table/035943993d33434d9f23a29c0daf42d9
2024-11-20T19:42:23,605 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/info/8232b9c7bd6f425d88e7658d847e4075 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/info/8232b9c7bd6f425d88e7658d847e4075
2024-11-20T19:42:23,613 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/info/8232b9c7bd6f425d88e7658d847e4075, entries=10, sequenceid=11, filesize=6.9 K
2024-11-20T19:42:23,614 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/ns/664db17e281f4d0b973f36b2e3e2ed8f as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/ns/664db17e281f4d0b973f36b2e3e2ed8f
2024-11-20T19:42:23,621 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/ns/664db17e281f4d0b973f36b2e3e2ed8f, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T19:42:23,622 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/.tmp/table/035943993d33434d9f23a29c0daf42d9 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/table/035943993d33434d9f23a29c0daf42d9
2024-11-20T19:42:23,630 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/table/035943993d33434d9f23a29c0daf42d9, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T19:42:23,632 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 528ms, sequenceid=11, compaction requested=false
2024-11-20T19:42:23,638 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T19:42:23,639 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:42:23,639 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:23,639 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131743102Running coprocessor pre-close hooks at 1732131743102Disabling compacts and flushes for region at 1732131743102Disabling writes for close at 1732131743103 (+1 ms)Obtaining lock to block concurrent updates at 1732131743103Preparing flush snapshotting stores in 1588230740 at 1732131743103Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732131743104 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732131743105 (+1 ms)Flushing 1588230740/info: creating writer at 1732131743105Flushing 1588230740/info: appending metadata at 1732131743128 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732131743128Flushing 1588230740/ns: creating writer at 1732131743144 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732131743159 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732131743159Flushing 1588230740/table: creating writer at 1732131743576 (+417 ms)Flushing 1588230740/table: appending metadata at 1732131743590 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732131743590Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c0530d5: reopening flushed file at 1732131743604 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c9a64be: reopening flushed file at 1732131743613 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4e5e72: reopening flushed file at 1732131743621 (+8 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 528ms, sequenceid=11, compaction requested=false at 1732131743632 (+11 ms)Writing region close event to WAL at 1732131743633 (+1 ms)Running coprocessor post-close hooks at 1732131743639 (+6 ms)Closed at 1732131743639
2024-11-20T19:42:23,640 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:23,704 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,46533,1732131647812; all regions closed.
2024-11-20T19:42:23,706 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,707 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,707 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,707 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,707 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741834_1010 (size=3066)
2024-11-20T19:42:23,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741834_1010 (size=3066)
2024-11-20T19:42:23,718 DEBUG [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs
2024-11-20T19:42:23,718 INFO  [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C46533%2C1732131647812.meta:.meta(num 1732131649714)
2024-11-20T19:42:23,718 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,718 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,718 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,718 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,719 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741847_1023 (size=12695)
2024-11-20T19:42:23,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741847_1023 (size=12695)
2024-11-20T19:42:23,724 DEBUG [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/oldWALs
2024-11-20T19:42:23,724 INFO  [RS:0;e14ef07817f4:46533 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C46533%2C1732131647812:(num 1732131722818)
2024-11-20T19:42:23,724 DEBUG [RS:0;e14ef07817f4:46533 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:23,724 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:42:23,724 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:42:23,724 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T19:42:23,725 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:42:23,725 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:42:23,725 INFO  [RS:0;e14ef07817f4:46533 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46533
2024-11-20T19:42:23,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,46533,1732131647812
2024-11-20T19:42:23,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:42:23,739 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:42:23,750 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,46533,1732131647812]
2024-11-20T19:42:23,760 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,46533,1732131647812 already deleted, retry=false
2024-11-20T19:42:23,760 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,46533,1732131647812 expired; onlineServers=0
2024-11-20T19:42:23,760 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,43333,1732131647044' *****
2024-11-20T19:42:23,761 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:42:23,761 INFO  [M:0;e14ef07817f4:43333 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:42:23,761 INFO  [M:0;e14ef07817f4:43333 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:42:23,761 DEBUG [M:0;e14ef07817f4:43333 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:42:23,762 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:42:23,762 DEBUG [M:0;e14ef07817f4:43333 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:42:23,762 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131648913 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131648913,5,FailOnTimeoutGroup]
2024-11-20T19:42:23,762 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131648914 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131648914,5,FailOnTimeoutGroup]
2024-11-20T19:42:23,762 INFO  [M:0;e14ef07817f4:43333 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:42:23,763 INFO  [M:0;e14ef07817f4:43333 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:42:23,763 DEBUG [M:0;e14ef07817f4:43333 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:42:23,763 INFO  [M:0;e14ef07817f4:43333 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:42:23,763 INFO  [M:0;e14ef07817f4:43333 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:42:23,765 INFO  [M:0;e14ef07817f4:43333 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:42:23,765 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:42:23,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:42:23,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:23,771 DEBUG [M:0;e14ef07817f4:43333 {}] zookeeper.ZKUtil(347): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:42:23,771 WARN  [M:0;e14ef07817f4:43333 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:42:23,772 INFO  [M:0;e14ef07817f4:43333 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/.lastflushedseqids
2024-11-20T19:42:23,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741854_1030 (size=130)
2024-11-20T19:42:23,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741854_1030 (size=130)
2024-11-20T19:42:23,786 INFO  [M:0;e14ef07817f4:43333 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:42:23,786 INFO  [M:0;e14ef07817f4:43333 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:42:23,787 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:42:23,787 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:23,787 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:23,787 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:42:23,787 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:23,787 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-20T19:42:23,804 DEBUG [M:0;e14ef07817f4:43333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe5caa1521b442b884aed7016dcab9c1 is 82, key is hbase:meta,,1/info:regioninfo/1732131649786/Put/seqid=0
2024-11-20T19:42:23,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741855_1031 (size=5672)
2024-11-20T19:42:23,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741855_1031 (size=5672)
2024-11-20T19:42:23,811 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe5caa1521b442b884aed7016dcab9c1
2024-11-20T19:42:23,830 DEBUG [M:0;e14ef07817f4:43333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2df094631e16416983d0b8d1573fce4f is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732131650633/Put/seqid=0
2024-11-20T19:42:23,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741856_1032 (size=6248)
2024-11-20T19:42:23,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741856_1032 (size=6248)
2024-11-20T19:42:23,837 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2df094631e16416983d0b8d1573fce4f
2024-11-20T19:42:23,843 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2df094631e16416983d0b8d1573fce4f
2024-11-20T19:42:23,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:23,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46533-0x10159c366a10001, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:23,850 INFO  [RS:0;e14ef07817f4:46533 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:42:23,850 INFO  [RS:0;e14ef07817f4:46533 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,46533,1732131647812; zookeeper connection closed.
2024-11-20T19:42:23,851 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46e1b389 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46e1b389
2024-11-20T19:42:23,851 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:42:23,860 DEBUG [M:0;e14ef07817f4:43333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/927194d82ddd473f9a011d8da94a1cef is 69, key is e14ef07817f4,46533,1732131647812/rs:state/1732131648963/Put/seqid=0
2024-11-20T19:42:23,866 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741857_1033 (size=5156)
2024-11-20T19:42:23,866 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741857_1033 (size=5156)
2024-11-20T19:42:23,867 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/927194d82ddd473f9a011d8da94a1cef
2024-11-20T19:42:23,888 DEBUG [M:0;e14ef07817f4:43333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e10acad00a4d4d48ad52c7a2c29502bf is 52, key is load_balancer_on/state:d/1732131650123/Put/seqid=0
2024-11-20T19:42:23,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741858_1034 (size=5056)
2024-11-20T19:42:23,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741858_1034 (size=5056)
2024-11-20T19:42:23,898 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e10acad00a4d4d48ad52c7a2c29502bf
2024-11-20T19:42:23,905 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fe5caa1521b442b884aed7016dcab9c1 as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe5caa1521b442b884aed7016dcab9c1
2024-11-20T19:42:23,911 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fe5caa1521b442b884aed7016dcab9c1, entries=8, sequenceid=59, filesize=5.5 K
2024-11-20T19:42:23,913 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2df094631e16416983d0b8d1573fce4f as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2df094631e16416983d0b8d1573fce4f
2024-11-20T19:42:23,919 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2df094631e16416983d0b8d1573fce4f
2024-11-20T19:42:23,920 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2df094631e16416983d0b8d1573fce4f, entries=6, sequenceid=59, filesize=6.1 K
2024-11-20T19:42:23,921 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/927194d82ddd473f9a011d8da94a1cef as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/927194d82ddd473f9a011d8da94a1cef
2024-11-20T19:42:23,928 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/927194d82ddd473f9a011d8da94a1cef, entries=1, sequenceid=59, filesize=5.0 K
2024-11-20T19:42:23,929 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e10acad00a4d4d48ad52c7a2c29502bf as hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e10acad00a4d4d48ad52c7a2c29502bf
2024-11-20T19:42:23,935 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e10acad00a4d4d48ad52c7a2c29502bf, entries=1, sequenceid=59, filesize=4.9 K
2024-11-20T19:42:23,936 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=59, compaction requested=false
2024-11-20T19:42:23,938 INFO  [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:23,938 DEBUG [M:0;e14ef07817f4:43333 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131743786Disabling compacts and flushes for region at 1732131743786Disabling writes for close at 1732131743787 (+1 ms)Obtaining lock to block concurrent updates at 1732131743787Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131743787Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1732131743787Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131743788 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131743788Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131743804 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131743804Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131743817 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131743830 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131743830Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131743843 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131743860 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131743860Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131743873 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131743887 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131743888 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eccb927: reopening flushed file at 1732131743903 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f5de74: reopening flushed file at 1732131743912 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@384eb1fd: reopening flushed file at 1732131743920 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51479084: reopening flushed file at 1732131743928 (+8 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=59, compaction requested=false at 1732131743936 (+8 ms)Writing region close event to WAL at 1732131743938 (+2 ms)Closed at 1732131743938
2024-11-20T19:42:23,939 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,939 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,939 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,939 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,939 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:23,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35233 is added to blk_1073741830_1006 (size=27985)
2024-11-20T19:42:23,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38433 is added to blk_1073741830_1006 (size=27985)
2024-11-20T19:42:23,942 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:42:23,942 INFO  [M:0;e14ef07817f4:43333 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:42:23,943 INFO  [M:0;e14ef07817f4:43333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43333
2024-11-20T19:42:23,943 INFO  [M:0;e14ef07817f4:43333 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:42:24,049 INFO  [M:0;e14ef07817f4:43333 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:42:24,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:24,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43333-0x10159c366a10000, quorum=127.0.0.1:60731, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:24,057 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:24,059 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:24,059 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:24,059 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:24,059 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:24,062 WARN  [BP-1427809238-172.17.0.3-1732131642807 heartbeating to localhost/127.0.0.1:43327 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:24,062 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:24,062 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:24,062 WARN  [BP-1427809238-172.17.0.3-1732131642807 heartbeating to localhost/127.0.0.1:43327 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1427809238-172.17.0.3-1732131642807 (Datanode Uuid 503ae475-45e5-4e34-91ed-55aab59f4075) service to localhost/127.0.0.1:43327
2024-11-20T19:42:24,063 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data3/current/BP-1427809238-172.17.0.3-1732131642807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:24,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data4/current/BP-1427809238-172.17.0.3-1732131642807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:24,064 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:24,066 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:24,066 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:24,066 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:24,067 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:24,067 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:24,068 WARN  [BP-1427809238-172.17.0.3-1732131642807 heartbeating to localhost/127.0.0.1:43327 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:24,068 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:24,068 WARN  [BP-1427809238-172.17.0.3-1732131642807 heartbeating to localhost/127.0.0.1:43327 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1427809238-172.17.0.3-1732131642807 (Datanode Uuid b1a0a592-fce8-4cf4-b45c-a2b59b4b2d66) service to localhost/127.0.0.1:43327
2024-11-20T19:42:24,068 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:24,069 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data1/current/BP-1427809238-172.17.0.3-1732131642807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:24,069 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/cluster_f93aa0aa-46a0-1c68-61cd-4194fbc643e0/data/data2/current/BP-1427809238-172.17.0.3-1732131642807 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:24,070 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:24,078 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:42:24,079 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:24,079 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:24,079 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:24,080 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:24,087 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:42:24,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:42:24,128 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12)
Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/e14ef07817f4:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43327 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43327 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: master/e14ef07817f4:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43327 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6f391bc8
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43327
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43327
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:43327
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43327
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/e14ef07817f4:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43327
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/e14ef07817f4:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=75 (was 150), ProcessCount=11 (was 11), AvailableMemoryMB=7855 (was 8405)
2024-11-20T19:42:24,135 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=75, ProcessCount=11, AvailableMemoryMB=7855
2024-11-20T19:42:24,135 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.log.dir so I do NOT create it in target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d6c0f40c-5e4c-8c93-a658-8b3b35a947a2/hadoop.tmp.dir so I do NOT create it in target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c, deleteOnExit=true
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/test.cache.data in system properties and HBase conf
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:42:24,136 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:42:24,137 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:42:24,137 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:42:24,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:42:24,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:42:24,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:42:24,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:42:24,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:42:24,150 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:42:24,547 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:24,554 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:24,555 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:24,555 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:24,555 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:24,557 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:24,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:24,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:24,653 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/java.io.tmpdir/jetty-localhost-42113-hadoop-hdfs-3_4_1-tests_jar-_-any-10982044074523588270/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:42:24,654 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:42113}
2024-11-20T19:42:24,654 INFO  [Time-limited test {}] server.Server(415): Started @103728ms
2024-11-20T19:42:24,666 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:42:24,926 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:24,930 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:24,931 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:24,931 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:24,931 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:24,932 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:24,932 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:25,038 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66b9e907{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/java.io.tmpdir/jetty-localhost-41565-hadoop-hdfs-3_4_1-tests_jar-_-any-15272477407330162182/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:25,039 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:41565}
2024-11-20T19:42:25,039 INFO  [Time-limited test {}] server.Server(415): Started @104113ms
2024-11-20T19:42:25,041 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:25,050 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:42:25,077 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:25,081 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:25,081 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:25,081 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:25,081 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:25,082 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:25,082 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:25,177 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c17f9e5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/java.io.tmpdir/jetty-localhost-37607-hadoop-hdfs-3_4_1-tests_jar-_-any-6883928552429181382/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:25,177 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:37607}
2024-11-20T19:42:25,177 INFO  [Time-limited test {}] server.Server(415): Started @104251ms
2024-11-20T19:42:25,179 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:26,287 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data1/current/BP-125219072-172.17.0.3-1732131744162/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:26,287 WARN  [Thread-455 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data2/current/BP-125219072-172.17.0.3-1732131744162/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:26,309 WARN  [Thread-418 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:26,311 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa000ab370c168220 with lease ID 0x7c214c247afc84f3: Processing first storage report for DS-c01b5329-88f0-405b-aca6-d49e3b7632cb from datanode DatanodeRegistration(127.0.0.1:40071, datanodeUuid=b88418df-8060-4194-8b7f-f0506a99650e, infoPort=43153, infoSecurePort=0, ipcPort=44013, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162)
2024-11-20T19:42:26,312 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa000ab370c168220 with lease ID 0x7c214c247afc84f3: from storage DS-c01b5329-88f0-405b-aca6-d49e3b7632cb node DatanodeRegistration(127.0.0.1:40071, datanodeUuid=b88418df-8060-4194-8b7f-f0506a99650e, infoPort=43153, infoSecurePort=0, ipcPort=44013, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:26,312 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa000ab370c168220 with lease ID 0x7c214c247afc84f3: Processing first storage report for DS-5cdf76ce-8fb3-49be-83ad-a82c0a506b87 from datanode DatanodeRegistration(127.0.0.1:40071, datanodeUuid=b88418df-8060-4194-8b7f-f0506a99650e, infoPort=43153, infoSecurePort=0, ipcPort=44013, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162)
2024-11-20T19:42:26,312 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa000ab370c168220 with lease ID 0x7c214c247afc84f3: from storage DS-5cdf76ce-8fb3-49be-83ad-a82c0a506b87 node DatanodeRegistration(127.0.0.1:40071, datanodeUuid=b88418df-8060-4194-8b7f-f0506a99650e, infoPort=43153, infoSecurePort=0, ipcPort=44013, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:26,427 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data3/current/BP-125219072-172.17.0.3-1732131744162/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:26,427 WARN  [Thread-466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data4/current/BP-125219072-172.17.0.3-1732131744162/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:26,441 WARN  [Thread-441 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:26,444 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e52c52840048e20 with lease ID 0x7c214c247afc84f4: Processing first storage report for DS-f18600ce-ab32-42e8-b9dc-92394b9b25d4 from datanode DatanodeRegistration(127.0.0.1:43187, datanodeUuid=cf6a2cf8-d27a-4821-9fff-8967cb54eee5, infoPort=35303, infoSecurePort=0, ipcPort=41349, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162)
2024-11-20T19:42:26,444 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e52c52840048e20 with lease ID 0x7c214c247afc84f4: from storage DS-f18600ce-ab32-42e8-b9dc-92394b9b25d4 node DatanodeRegistration(127.0.0.1:43187, datanodeUuid=cf6a2cf8-d27a-4821-9fff-8967cb54eee5, infoPort=35303, infoSecurePort=0, ipcPort=41349, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:42:26,444 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e52c52840048e20 with lease ID 0x7c214c247afc84f4: Processing first storage report for DS-797f8d42-a162-4f47-900d-ad8802142cf0 from datanode DatanodeRegistration(127.0.0.1:43187, datanodeUuid=cf6a2cf8-d27a-4821-9fff-8967cb54eee5, infoPort=35303, infoSecurePort=0, ipcPort=41349, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162)
2024-11-20T19:42:26,444 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e52c52840048e20 with lease ID 0x7c214c247afc84f4: from storage DS-797f8d42-a162-4f47-900d-ad8802142cf0 node DatanodeRegistration(127.0.0.1:43187, datanodeUuid=cf6a2cf8-d27a-4821-9fff-8967cb54eee5, infoPort=35303, infoSecurePort=0, ipcPort=41349, storageInfo=lv=-57;cid=testClusterID;nsid=1210396030;c=1732131744162), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:26,523 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14
2024-11-20T19:42:26,528 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/zookeeper_0, clientPort=59563, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:42:26,529 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59563
2024-11-20T19:42:26,529 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,530 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:42:26,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:42:26,541 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7 with version=8
2024-11-20T19:42:26,542 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:42:26,544 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:42:26,545 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:32959
2024-11-20T19:42:26,547 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32959 connecting to ZooKeeper ensemble=127.0.0.1:59563
2024-11-20T19:42:26,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:329590x0, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:42:26,604 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32959-0x10159c4ee8a0000 connected
2024-11-20T19:42:26,687 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,692 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:26,696 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7, hbase.cluster.distributed=false
2024-11-20T19:42:26,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:42:26,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32959
2024-11-20T19:42:26,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32959
2024-11-20T19:42:26,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32959
2024-11-20T19:42:26,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32959
2024-11-20T19:42:26,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32959
2024-11-20T19:42:26,714 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:42:26,715 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:42:26,716 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46257
2024-11-20T19:42:26,718 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46257 connecting to ZooKeeper ensemble=127.0.0.1:59563
2024-11-20T19:42:26,718 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,720 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462570x0, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:42:26,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46257-0x10159c4ee8a0001 connected
2024-11-20T19:42:26,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:26,729 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:42:26,730 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:42:26,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:42:26,732 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:42:26,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46257
2024-11-20T19:42:26,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46257
2024-11-20T19:42:26,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46257
2024-11-20T19:42:26,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46257
2024-11-20T19:42:26,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46257
2024-11-20T19:42:26,746 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:32959
2024-11-20T19:42:26,746 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,32959,1732131746543
2024-11-20T19:42:26,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:26,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:26,759 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,32959,1732131746543
2024-11-20T19:42:26,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:42:26,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:26,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:26,771 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:42:26,772 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,32959,1732131746543 from backup master directory
2024-11-20T19:42:26,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,32959,1732131746543
2024-11-20T19:42:26,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:26,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:26,781 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:42:26,781 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,32959,1732131746543
2024-11-20T19:42:26,789 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/hbase.id] with ID: 76ece6b2-96ea-42b3-97f4-bb4677c3d3d1
2024-11-20T19:42:26,790 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/.tmp/hbase.id
2024-11-20T19:42:26,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:42:26,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:42:26,798 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/.tmp/hbase.id]:[hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/hbase.id]
2024-11-20T19:42:26,814 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:26,815 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:42:26,817 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-20T19:42:26,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:26,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:26,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:42:26,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:42:26,835 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:42:26,836 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:42:26,836 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:26,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:42:26,846 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:42:27,251 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store
2024-11-20T19:42:27,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:42:27,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:42:27,265 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:27,265 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:27,265 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131747265Disabling compacts and flushes for region at 1732131747265Disabling writes for close at 1732131747265Writing region close event to WAL at 1732131747265Closed at 1732131747265
2024-11-20T19:42:27,267 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/.initializing
2024-11-20T19:42:27,267 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/WALs/e14ef07817f4,32959,1732131746543
2024-11-20T19:42:27,272 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C32959%2C1732131746543, suffix=, logDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/WALs/e14ef07817f4,32959,1732131746543, archiveDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/oldWALs, maxLogs=10
2024-11-20T19:42:27,272 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C32959%2C1732131746543.1732131747272
2024-11-20T19:42:27,280 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/WALs/e14ef07817f4,32959,1732131746543/e14ef07817f4%2C32959%2C1732131746543.1732131747272
2024-11-20T19:42:27,286 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35303:35303),(127.0.0.1/127.0.0.1:43153:43153)]
2024-11-20T19:42:27,287 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:42:27,287 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:27,288 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,288 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,290 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:27,291 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:42:27,291 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,292 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:27,292 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:27,294 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:42:27,294 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,294 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:27,295 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:27,297 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:42:27,297 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,298 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:27,298 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:27,299 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:42:27,299 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,300 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:27,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,301 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,301 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,302 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,302 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,303 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:42:27,304 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:27,307 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:42:27,307 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754785, jitterRate=-0.04024180769920349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:42:27,308 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131747288Initializing all the Stores at 1732131747289 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131747289Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131747289Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131747289Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131747290 (+1 ms)Cleaning up temporary data from old regions at 1732131747302 (+12 ms)Region opened successfully at 1732131747308 (+6 ms)
2024-11-20T19:42:27,309 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:42:27,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78be6dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:42:27,314 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:42:27,314 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:42:27,314 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:42:27,314 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:42:27,315 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:42:27,315 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:42:27,315 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:42:27,318 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:42:27,319 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:42:27,370 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:42:27,370 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:42:27,372 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:42:27,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:42:27,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:42:27,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-20T19:42:27,381 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:42:27,381 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:42:27,383 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:42:27,391 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:42:27,393 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:42:27,402 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:42:27,407 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:42:27,412 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:42:27,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:27,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:27,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,424 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,32959,1732131746543, sessionid=0x10159c4ee8a0000, setting cluster-up flag (Was=false)
2024-11-20T19:42:27,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,476 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:42:27,480 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,32959,1732131746543
2024-11-20T19:42:27,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:27,538 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:42:27,540 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,32959,1732131746543
2024-11-20T19:42:27,542 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:42:27,545 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:27,545 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:42:27,545 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:42:27,546 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,32959,1732131746543
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:42:27,549 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:27,549 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:27,549 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:27,549 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:27,549 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:42:27,550 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,550 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:42:27,550 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131777551
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:42:27,551 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:42:27,552 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,552 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:42:27,552 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:27,552 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:42:27,552 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:42:27,552 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:42:27,553 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:42:27,553 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:42:27,553 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131747553,5,FailOnTimeoutGroup]
2024-11-20T19:42:27,553 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131747553,5,FailOnTimeoutGroup]
2024-11-20T19:42:27,553 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,554 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,554 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:42:27,554 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,554 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,554 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:42:27,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:42:27,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:42:27,563 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:42:27,563 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7
2024-11-20T19:42:27,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:42:27,571 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:42:27,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:27,573 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:42:27,575 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:42:27,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,575 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:27,576 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:42:27,577 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:42:27,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,578 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:27,578 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:42:27,579 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:42:27,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:27,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:42:27,582 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:42:27,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:27,582 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:27,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:42:27,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740
2024-11-20T19:42:27,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740
2024-11-20T19:42:27,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:42:27,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:42:27,585 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:42:27,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:42:27,589 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:42:27,589 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773269, jitterRate=-0.0167379230260849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:42:27,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131747572Initializing all the Stores at 1732131747573 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131747573Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131747573Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131747573Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131747573Cleaning up temporary data from old regions at 1732131747585 (+12 ms)Region opened successfully at 1732131747590 (+5 ms)
2024-11-20T19:42:27,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:42:27,590 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:42:27,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:42:27,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:42:27,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:42:27,591 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:27,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131747590Disabling compacts and flushes for region at 1732131747590Disabling writes for close at 1732131747591 (+1 ms)Writing region close event to WAL at 1732131747591Closed at 1732131747591
2024-11-20T19:42:27,593 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:27,593 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:42:27,593 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:42:27,595 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:42:27,596 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:42:27,636 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(746): ClusterId : 76ece6b2-96ea-42b3-97f4-bb4677c3d3d1
2024-11-20T19:42:27,636 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:42:27,645 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:42:27,645 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:42:27,655 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:42:27,656 DEBUG [RS:0;e14ef07817f4:46257 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eeaca7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:42:27,668 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:46257
2024-11-20T19:42:27,668 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:42:27,668 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:42:27,668 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:42:27,669 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,32959,1732131746543 with port=46257, startcode=1732131746714
2024-11-20T19:42:27,669 DEBUG [RS:0;e14ef07817f4:46257 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:42:27,671 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58929, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:42:27,672 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,672 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32959 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,674 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7
2024-11-20T19:42:27,674 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41931
2024-11-20T19:42:27,674 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:42:27,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:42:27,686 DEBUG [RS:0;e14ef07817f4:46257 {}] zookeeper.ZKUtil(111): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,687 WARN  [RS:0;e14ef07817f4:46257 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:42:27,687 INFO  [RS:0;e14ef07817f4:46257 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:27,687 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/WALs/e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,687 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,46257,1732131746714]
2024-11-20T19:42:27,691 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:42:27,693 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:42:27,694 INFO  [RS:0;e14ef07817f4:46257 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:42:27,694 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,694 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:42:27,695 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:42:27,695 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:42:27,695 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:27,696 DEBUG [RS:0;e14ef07817f4:46257 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:27,696 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,696 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,696 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,697 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,697 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,697 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,46257,1732131746714-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:42:27,711 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:42:27,711 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,46257,1732131746714-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,712 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,712 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.Replication(171): e14ef07817f4,46257,1732131746714 started
2024-11-20T19:42:27,726 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:27,727 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,46257,1732131746714, RpcServer on e14ef07817f4/172.17.0.3:46257, sessionid=0x10159c4ee8a0001
2024-11-20T19:42:27,727 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:42:27,727 DEBUG [RS:0;e14ef07817f4:46257 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,727 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,46257,1732131746714'
2024-11-20T19:42:27,727 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:42:27,728 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:42:27,728 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:42:27,728 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:42:27,728 DEBUG [RS:0;e14ef07817f4:46257 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,46257,1732131746714
2024-11-20T19:42:27,729 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,46257,1732131746714'
2024-11-20T19:42:27,729 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:42:27,729 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:42:27,730 DEBUG [RS:0;e14ef07817f4:46257 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:42:27,730 INFO  [RS:0;e14ef07817f4:46257 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:42:27,730 INFO  [RS:0;e14ef07817f4:46257 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:42:27,746 WARN  [e14ef07817f4:32959 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:42:27,833 INFO  [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C46257%2C1732131746714, suffix=, logDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/WALs/e14ef07817f4,46257,1732131746714, archiveDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/oldWALs, maxLogs=32
2024-11-20T19:42:27,838 INFO  [RS:0;e14ef07817f4:46257 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46257%2C1732131746714.1732131747837
2024-11-20T19:42:27,846 INFO  [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/WALs/e14ef07817f4,46257,1732131746714/e14ef07817f4%2C46257%2C1732131746714.1732131747837
2024-11-20T19:42:27,848 DEBUG [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43153:43153),(127.0.0.1/127.0.0.1:35303:35303)]
2024-11-20T19:42:27,997 DEBUG [e14ef07817f4:32959 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:42:27,998 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,46257,1732131746714
2024-11-20T19:42:28,002 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,46257,1732131746714, state=OPENING
2024-11-20T19:42:28,012 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:42:28,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:28,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:28,023 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:42:28,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:28,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:28,023 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,46257,1732131746714}]
2024-11-20T19:42:28,178 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:42:28,184 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:42:28,192 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:42:28,192 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:28,195 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C46257%2C1732131746714.meta, suffix=.meta, logDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/WALs/e14ef07817f4,46257,1732131746714, archiveDir=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/oldWALs, maxLogs=32
2024-11-20T19:42:28,197 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C46257%2C1732131746714.meta.1732131748197.meta
2024-11-20T19:42:28,203 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/WALs/e14ef07817f4,46257,1732131746714/e14ef07817f4%2C46257%2C1732131746714.meta.1732131748197.meta
2024-11-20T19:42:28,204 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43153:43153),(127.0.0.1/127.0.0.1:35303:35303)]
2024-11-20T19:42:28,208 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:42:28,209 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:42:28,209 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:42:28,211 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:42:28,212 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:42:28,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:28,212 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:28,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:42:28,213 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:42:28,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:28,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:28,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:42:28,215 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:42:28,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:28,216 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:28,216 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:42:28,217 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:42:28,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:28,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:28,218 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:42:28,218 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740
2024-11-20T19:42:28,220 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740
2024-11-20T19:42:28,221 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:42:28,221 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:42:28,222 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:42:28,223 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:42:28,224 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=746523, jitterRate=-0.050748005509376526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:42:28,224 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:42:28,225 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131748209Writing region info on filesystem at 1732131748209Initializing all the Stores at 1732131748210 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131748210Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131748211 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131748211Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131748211Cleaning up temporary data from old regions at 1732131748221 (+10 ms)Running coprocessor post-open hooks at 1732131748225 (+4 ms)Region opened successfully at 1732131748225
2024-11-20T19:42:28,227 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131748178
2024-11-20T19:42:28,229 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:42:28,230 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:42:28,231 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,46257,1732131746714
2024-11-20T19:42:28,232 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,46257,1732131746714, state=OPEN
2024-11-20T19:42:28,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:42:28,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:42:28,304 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,46257,1732131746714
2024-11-20T19:42:28,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:28,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:28,311 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:42:28,311 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,46257,1732131746714 in 281 msec
2024-11-20T19:42:28,315 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:42:28,316 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 718 msec
2024-11-20T19:42:28,317 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:28,317 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:42:28,319 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:42:28,319 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,46257,1732131746714, seqNum=-1]
2024-11-20T19:42:28,320 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:42:28,321 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55381, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:42:28,330 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 785 msec
2024-11-20T19:42:28,331 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131748330, completionTime=-1
2024-11-20T19:42:28,331 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:42:28,331 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:42:28,333 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:42:28,333 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131808333
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732131868334
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:32959, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,336 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.558sec
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:42:28,339 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:42:28,342 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:42:28,342 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:42:28,342 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,32959,1732131746543-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:28,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57927b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:28,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,32959,-1 for getting cluster id
2024-11-20T19:42:28,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:42:28,442 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '76ece6b2-96ea-42b3-97f4-bb4677c3d3d1'
2024-11-20T19:42:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:42:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "76ece6b2-96ea-42b3-97f4-bb4677c3d3d1"

2024-11-20T19:42:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d34ab95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,32959,-1]
2024-11-20T19:42:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:42:28,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,448 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:42:28,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa3a1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:28,450 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:42:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,46257,1732131746714, seqNum=-1]
2024-11-20T19:42:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:42:28,454 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:42:28,456 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,32959,1732131746543
2024-11-20T19:42:28,456 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:28,459 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:42:28,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:42:28,460 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:42:28,460 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:28,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,460 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:42:28,460 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:42:28,460 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=361581943, stopped=false
2024-11-20T19:42:28,460 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,32959,1732131746543
2024-11-20T19:42:28,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:28,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:28,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:28,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:28,475 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:42:28,476 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:42:28,476 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:28,476 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:28,476 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:28,476 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,46257,1732131746714' *****
2024-11-20T19:42:28,476 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:42:28,477 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,46257,1732131746714
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:46257.
2024-11-20T19:42:28,477 DEBUG [RS:0;e14ef07817f4:46257 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:42:28,477 DEBUG [RS:0;e14ef07817f4:46257 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:42:28,477 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:42:28,478 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T19:42:28,478 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:42:28,478 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:42:28,478 DEBUG [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T19:42:28,478 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:42:28,478 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:42:28,478 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:42:28,478 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:42:28,478 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-20T19:42:28,494 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/.tmp/ns/20ac196f179d4c72b264de7312b5dfa4 is 43, key is default/ns:d/1732131748322/Put/seqid=0
2024-11-20T19:42:28,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741835_1011 (size=5153)
2024-11-20T19:42:28,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741835_1011 (size=5153)
2024-11-20T19:42:28,500 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/.tmp/ns/20ac196f179d4c72b264de7312b5dfa4
2024-11-20T19:42:28,507 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/.tmp/ns/20ac196f179d4c72b264de7312b5dfa4 as hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/ns/20ac196f179d4c72b264de7312b5dfa4
2024-11-20T19:42:28,513 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/ns/20ac196f179d4c72b264de7312b5dfa4, entries=2, sequenceid=6, filesize=5.0 K
2024-11-20T19:42:28,515 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false
2024-11-20T19:42:28,515 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:42:28,520 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-20T19:42:28,521 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:42:28,521 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:28,521 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131748478Running coprocessor pre-close hooks at 1732131748478Disabling compacts and flushes for region at 1732131748478Disabling writes for close at 1732131748478Obtaining lock to block concurrent updates at 1732131748478Preparing flush snapshotting stores in 1588230740 at 1732131748478Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732131748479 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732131748479Flushing 1588230740/ns: creating writer at 1732131748479Flushing 1588230740/ns: appending metadata at 1732131748493 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732131748493Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2de928c2: reopening flushed file at 1732131748506 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false at 1732131748515 (+9 ms)Writing region close event to WAL at 1732131748516 (+1 ms)Running coprocessor post-close hooks at 1732131748520 (+4 ms)Closed at 1732131748521 (+1 ms)
2024-11-20T19:42:28,521 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:28,580 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:28,584 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:28,678 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,46257,1732131746714; all regions closed.
2024-11-20T19:42:28,679 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,679 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,679 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,680 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,680 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741834_1010 (size=1152)
2024-11-20T19:42:28,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741834_1010 (size=1152)
2024-11-20T19:42:28,687 DEBUG [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/oldWALs
2024-11-20T19:42:28,687 INFO  [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C46257%2C1732131746714.meta:.meta(num 1732131748197)
2024-11-20T19:42:28,688 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,688 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,689 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,689 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,689 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741833_1009 (size=93)
2024-11-20T19:42:28,691 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741833_1009 (size=93)
2024-11-20T19:42:28,694 DEBUG [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/oldWALs
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C46257%2C1732131746714:(num 1732131747837)
2024-11-20T19:42:28,694 DEBUG [RS:0;e14ef07817f4:46257 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:42:28,694 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:42:28,694 INFO  [RS:0;e14ef07817f4:46257 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46257
2024-11-20T19:42:28,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,46257,1732131746714
2024-11-20T19:42:28,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:42:28,707 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:42:28,718 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,46257,1732131746714]
2024-11-20T19:42:28,728 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,46257,1732131746714 already deleted, retry=false
2024-11-20T19:42:28,728 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,46257,1732131746714 expired; onlineServers=0
2024-11-20T19:42:28,728 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,32959,1732131746543' *****
2024-11-20T19:42:28,728 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:42:28,728 INFO  [M:0;e14ef07817f4:32959 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:42:28,729 INFO  [M:0;e14ef07817f4:32959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:42:28,729 DEBUG [M:0;e14ef07817f4:32959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:42:28,729 DEBUG [M:0;e14ef07817f4:32959 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:42:28,729 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:42:28,729 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131747553 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131747553,5,FailOnTimeoutGroup]
2024-11-20T19:42:28,729 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131747553 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131747553,5,FailOnTimeoutGroup]
2024-11-20T19:42:28,730 INFO  [M:0;e14ef07817f4:32959 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:42:28,730 INFO  [M:0;e14ef07817f4:32959 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:42:28,730 DEBUG [M:0;e14ef07817f4:32959 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:42:28,730 INFO  [M:0;e14ef07817f4:32959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:42:28,731 INFO  [M:0;e14ef07817f4:32959 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:42:28,731 INFO  [M:0;e14ef07817f4:32959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:42:28,731 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:42:28,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:42:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:28,739 DEBUG [M:0;e14ef07817f4:32959 {}] zookeeper.ZKUtil(347): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:42:28,739 WARN  [M:0;e14ef07817f4:32959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:42:28,740 INFO  [M:0;e14ef07817f4:32959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/.lastflushedseqids
2024-11-20T19:42:28,748 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741836_1012 (size=99)
2024-11-20T19:42:28,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741836_1012 (size=99)
2024-11-20T19:42:28,749 INFO  [M:0;e14ef07817f4:32959 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:42:28,749 INFO  [M:0;e14ef07817f4:32959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:42:28,749 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:42:28,750 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:28,750 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:28,750 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:42:28,750 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:28,750 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-20T19:42:28,770 DEBUG [M:0;e14ef07817f4:32959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51f4de51908141d6bfbe354baf0ecd40 is 82, key is hbase:meta,,1/info:regioninfo/1732131748230/Put/seqid=0
2024-11-20T19:42:28,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741837_1013 (size=5672)
2024-11-20T19:42:28,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741837_1013 (size=5672)
2024-11-20T19:42:28,777 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51f4de51908141d6bfbe354baf0ecd40
2024-11-20T19:42:28,798 DEBUG [M:0;e14ef07817f4:32959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dd1411e6a204108a66003845a1d90f7 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732131748329/Put/seqid=0
2024-11-20T19:42:28,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741838_1014 (size=5275)
2024-11-20T19:42:28,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741838_1014 (size=5275)
2024-11-20T19:42:28,804 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dd1411e6a204108a66003845a1d90f7
2024-11-20T19:42:28,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:28,818 INFO  [RS:0;e14ef07817f4:46257 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:42:28,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46257-0x10159c4ee8a0001, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:28,818 INFO  [RS:0;e14ef07817f4:46257 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,46257,1732131746714; zookeeper connection closed.
2024-11-20T19:42:28,818 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3261a6ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3261a6ac
2024-11-20T19:42:28,818 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:42:28,823 DEBUG [M:0;e14ef07817f4:32959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/308809752c7c43a7baeaaee31d3d117b is 69, key is e14ef07817f4,46257,1732131746714/rs:state/1732131747672/Put/seqid=0
2024-11-20T19:42:28,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741839_1015 (size=5156)
2024-11-20T19:42:28,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741839_1015 (size=5156)
2024-11-20T19:42:28,829 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/308809752c7c43a7baeaaee31d3d117b
2024-11-20T19:42:28,853 DEBUG [M:0;e14ef07817f4:32959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fe08158e3254c628bf51b33f872d185 is 52, key is load_balancer_on/state:d/1732131748458/Put/seqid=0
2024-11-20T19:42:28,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741840_1016 (size=5056)
2024-11-20T19:42:28,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741840_1016 (size=5056)
2024-11-20T19:42:28,859 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fe08158e3254c628bf51b33f872d185
2024-11-20T19:42:28,865 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51f4de51908141d6bfbe354baf0ecd40 as hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51f4de51908141d6bfbe354baf0ecd40
2024-11-20T19:42:28,872 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51f4de51908141d6bfbe354baf0ecd40, entries=8, sequenceid=29, filesize=5.5 K
2024-11-20T19:42:28,873 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dd1411e6a204108a66003845a1d90f7 as hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dd1411e6a204108a66003845a1d90f7
2024-11-20T19:42:28,879 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dd1411e6a204108a66003845a1d90f7, entries=3, sequenceid=29, filesize=5.2 K
2024-11-20T19:42:28,880 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/308809752c7c43a7baeaaee31d3d117b as hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/308809752c7c43a7baeaaee31d3d117b
2024-11-20T19:42:28,886 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/308809752c7c43a7baeaaee31d3d117b, entries=1, sequenceid=29, filesize=5.0 K
2024-11-20T19:42:28,888 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6fe08158e3254c628bf51b33f872d185 as hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6fe08158e3254c628bf51b33f872d185
2024-11-20T19:42:28,893 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41931/user/jenkins/test-data/3209faed-89b4-474b-5d7e-72dde42f8bb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6fe08158e3254c628bf51b33f872d185, entries=1, sequenceid=29, filesize=4.9 K
2024-11-20T19:42:28,895 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=29, compaction requested=false
2024-11-20T19:42:28,896 INFO  [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:28,896 DEBUG [M:0;e14ef07817f4:32959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131748749Disabling compacts and flushes for region at 1732131748749Disabling writes for close at 1732131748750 (+1 ms)Obtaining lock to block concurrent updates at 1732131748750Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131748750Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732131748750Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131748751 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131748751Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131748770 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131748770Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131748783 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131748797 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131748797Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131748809 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131748823 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131748823Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131748835 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131748852 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131748852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38102455: reopening flushed file at 1732131748864 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@671bb51c: reopening flushed file at 1732131748872 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3458208: reopening flushed file at 1732131748879 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ebcc7cc: reopening flushed file at 1732131748887 (+8 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 145ms, sequenceid=29, compaction requested=false at 1732131748895 (+8 ms)Writing region close event to WAL at 1732131748896 (+1 ms)Closed at 1732131748896
2024-11-20T19:42:28,897 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,897 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,897 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,897 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,897 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:28,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40071 is added to blk_1073741830_1006 (size=10311)
2024-11-20T19:42:28,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43187 is added to blk_1073741830_1006 (size=10311)
2024-11-20T19:42:28,900 INFO  [M:0;e14ef07817f4:32959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:42:28,900 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:42:28,901 INFO  [M:0;e14ef07817f4:32959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:32959
2024-11-20T19:42:28,901 INFO  [M:0;e14ef07817f4:32959 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:42:29,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:29,007 INFO  [M:0;e14ef07817f4:32959 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:42:29,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32959-0x10159c4ee8a0000, quorum=127.0.0.1:59563, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:42:29,010 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c17f9e5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:29,010 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:29,010 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:29,011 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:29,011 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:29,012 WARN  [BP-125219072-172.17.0.3-1732131744162 heartbeating to localhost/127.0.0.1:41931 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:29,012 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:29,012 WARN  [BP-125219072-172.17.0.3-1732131744162 heartbeating to localhost/127.0.0.1:41931 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125219072-172.17.0.3-1732131744162 (Datanode Uuid cf6a2cf8-d27a-4821-9fff-8967cb54eee5) service to localhost/127.0.0.1:41931
2024-11-20T19:42:29,012 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:29,013 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data3/current/BP-125219072-172.17.0.3-1732131744162 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:29,013 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data4/current/BP-125219072-172.17.0.3-1732131744162 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:29,013 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:29,018 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66b9e907{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:29,019 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:29,019 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:29,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:29,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:29,020 WARN  [BP-125219072-172.17.0.3-1732131744162 heartbeating to localhost/127.0.0.1:41931 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:29,020 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:29,020 WARN  [BP-125219072-172.17.0.3-1732131744162 heartbeating to localhost/127.0.0.1:41931 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-125219072-172.17.0.3-1732131744162 (Datanode Uuid b88418df-8060-4194-8b7f-f0506a99650e) service to localhost/127.0.0.1:41931
2024-11-20T19:42:29,020 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:29,021 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data1/current/BP-125219072-172.17.0.3-1732131744162 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:29,021 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/cluster_859b64a0-02fc-c30b-9923-29ec4677c97c/data/data2/current/BP-125219072-172.17.0.3-1732131744162 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:29,021 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:29,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:42:29,027 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:29,027 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:29,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:29,027 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:29,032 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.log.dir so I do NOT create it in target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6e36e69-95a3-afed-a44d-76bb71178b14/hadoop.tmp.dir so I do NOT create it in target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6, deleteOnExit=true
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/test.cache.data in system properties and HBase conf
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:42:29,048 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:42:29,049 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:42:29,049 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:42:29,050 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:42:29,062 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:42:29,096 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:42:29,101 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:29,111 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:29,113 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:29,113 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:29,427 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:29,432 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:29,433 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:29,433 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:29,434 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:29,434 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:29,435 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:29,435 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:29,528 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2606b08f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-39803-hadoop-hdfs-3_4_1-tests_jar-_-any-3656419514148780952/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:42:29,529 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:39803}
2024-11-20T19:42:29,529 INFO  [Time-limited test {}] server.Server(415): Started @108603ms
2024-11-20T19:42:29,540 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:42:29,697 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:42:29,808 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:29,812 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:29,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:29,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:29,813 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:29,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@180ba686{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:29,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21f2acf7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:29,906 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5afc739a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-37579-hadoop-hdfs-3_4_1-tests_jar-_-any-16727050554254370294/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:29,907 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@200d0f88{HTTP/1.1, (http/1.1)}{localhost:37579}
2024-11-20T19:42:29,907 INFO  [Time-limited test {}] server.Server(415): Started @108981ms
2024-11-20T19:42:29,908 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:29,934 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:29,938 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:29,941 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:29,941 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:29,941 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:29,941 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57ebe64c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:29,941 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2dc8ddff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:30,034 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48b0be64{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-44477-hadoop-hdfs-3_4_1-tests_jar-_-any-9503614201783079208/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:30,034 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4cae84f2{HTTP/1.1, (http/1.1)}{localhost:44477}
2024-11-20T19:42:30,034 INFO  [Time-limited test {}] server.Server(415): Started @109109ms
2024-11-20T19:42:30,036 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:31,353 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data1/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:31,353 WARN  [Thread-675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data2/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:31,377 WARN  [Thread-638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:31,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66f20f77d0192fe7 with lease ID 0x32e92df50b0f6209: Processing first storage report for DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707 from datanode DatanodeRegistration(127.0.0.1:32909, datanodeUuid=71812e8a-22fd-4421-a96c-a4d30768b529, infoPort=45335, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:31,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66f20f77d0192fe7 with lease ID 0x32e92df50b0f6209: from storage DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707 node DatanodeRegistration(127.0.0.1:32909, datanodeUuid=71812e8a-22fd-4421-a96c-a4d30768b529, infoPort=45335, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:42:31,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x66f20f77d0192fe7 with lease ID 0x32e92df50b0f6209: Processing first storage report for DS-9fc061c2-60ef-4ec1-97e3-da81b1b13848 from datanode DatanodeRegistration(127.0.0.1:32909, datanodeUuid=71812e8a-22fd-4421-a96c-a4d30768b529, infoPort=45335, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:31,380 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x66f20f77d0192fe7 with lease ID 0x32e92df50b0f6209: from storage DS-9fc061c2-60ef-4ec1-97e3-da81b1b13848 node DatanodeRegistration(127.0.0.1:32909, datanodeUuid=71812e8a-22fd-4421-a96c-a4d30768b529, infoPort=45335, infoSecurePort=0, ipcPort=46447, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:31,481 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data3/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:31,481 WARN  [Thread-686 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data4/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:31,499 WARN  [Thread-661 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:31,502 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefb07de82568d3b0 with lease ID 0x32e92df50b0f620a: Processing first storage report for DS-00acf580-b45e-483d-a65c-06dbb9accd8f from datanode DatanodeRegistration(127.0.0.1:39335, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=34365, infoSecurePort=0, ipcPort=35947, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:31,502 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefb07de82568d3b0 with lease ID 0x32e92df50b0f620a: from storage DS-00acf580-b45e-483d-a65c-06dbb9accd8f node DatanodeRegistration(127.0.0.1:39335, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=34365, infoSecurePort=0, ipcPort=35947, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:31,502 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefb07de82568d3b0 with lease ID 0x32e92df50b0f620a: Processing first storage report for DS-c9386add-398d-4307-8b7a-362613faee15 from datanode DatanodeRegistration(127.0.0.1:39335, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=34365, infoSecurePort=0, ipcPort=35947, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:31,502 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefb07de82568d3b0 with lease ID 0x32e92df50b0f620a: from storage DS-c9386add-398d-4307-8b7a-362613faee15 node DatanodeRegistration(127.0.0.1:39335, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=34365, infoSecurePort=0, ipcPort=35947, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:31,587 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867
2024-11-20T19:42:31,590 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/zookeeper_0, clientPort=62155, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:42:31,591 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62155
2024-11-20T19:42:31,591 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,593 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:42:31,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:42:31,603 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01 with version=8
2024-11-20T19:42:31,603 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:42:31,606 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:42:31,607 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:42:31,607 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39803
2024-11-20T19:42:31,610 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39803 connecting to ZooKeeper ensemble=127.0.0.1:62155
2024-11-20T19:42:31,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398030x0, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:42:31,673 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39803-0x10159c502510000 connected
2024-11-20T19:42:31,749 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,753 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,759 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:31,760 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01, hbase.cluster.distributed=false
2024-11-20T19:42:31,762 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:42:31,763 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39803
2024-11-20T19:42:31,763 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39803
2024-11-20T19:42:31,763 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39803
2024-11-20T19:42:31,763 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39803
2024-11-20T19:42:31,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39803
2024-11-20T19:42:31,777 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:42:31,778 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:42:31,779 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37815
2024-11-20T19:42:31,780 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37815 connecting to ZooKeeper ensemble=127.0.0.1:62155
2024-11-20T19:42:31,781 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,783 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378150x0, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:42:31,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378150x0, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:42:31,792 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37815-0x10159c502510001 connected
2024-11-20T19:42:31,792 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:42:31,792 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:42:31,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:42:31,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:42:31,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-20T19:42:31,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37815
2024-11-20T19:42:31,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37815
2024-11-20T19:42:31,798 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-20T19:42:31,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37815
2024-11-20T19:42:31,810 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:39803
2024-11-20T19:42:31,811 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,39803,1732131751605
2024-11-20T19:42:31,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:31,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:31,823 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,39803,1732131751605
2024-11-20T19:42:31,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:42:31,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:31,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:31,834 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:42:31,834 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,39803,1732131751605 from backup master directory
2024-11-20T19:42:31,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:31,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,39803,1732131751605
2024-11-20T19:42:31,843 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:42:31,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:42:31,843 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,39803,1732131751605
2024-11-20T19:42:31,848 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/hbase.id] with ID: 3d7e43a1-686f-4331-b0fe-3e0c3371338b
2024-11-20T19:42:31,848 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/.tmp/hbase.id
2024-11-20T19:42:31,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:42:31,858 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:42:31,859 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/.tmp/hbase.id]:[hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/hbase.id]
2024-11-20T19:42:31,873 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:31,873 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:42:31,875 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-20T19:42:31,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:31,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:31,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:42:31,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:42:31,973 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:42:31,974 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:42:31,974 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:31,984 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:42:31,984 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:42:31,985 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store
2024-11-20T19:42:31,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:42:31,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:42:31,995 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:31,995 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:42:31,995 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131751995Disabling compacts and flushes for region at 1732131751995Disabling writes for close at 1732131751995Writing region close event to WAL at 1732131751995Closed at 1732131751995
2024-11-20T19:42:31,996 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/.initializing
2024-11-20T19:42:31,997 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605
2024-11-20T19:42:31,999 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C39803%2C1732131751605, suffix=, logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605, archiveDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/oldWALs, maxLogs=10
2024-11-20T19:42:31,999 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C39803%2C1732131751605.1732131751999
2024-11-20T19:42:32,005 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999
2024-11-20T19:42:32,008 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34365:34365),(127.0.0.1/127.0.0.1:45335:45335)]
2024-11-20T19:42:32,008 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:42:32,009 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:32,009 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,009 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,011 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:32,013 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:42:32,013 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,013 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:32,014 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:32,015 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:42:32,015 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,016 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:32,016 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:32,018 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:42:32,018 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,019 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:32,019 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:42:32,020 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:42:32,021 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,021 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:32,021 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,022 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,023 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,024 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,024 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,025 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:42:32,027 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:42:32,030 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:42:32,030 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796044, jitterRate=0.012222766876220703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:42:32,032 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131752009Initializing all the Stores at 1732131752010 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131752010Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131752010Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131752010Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131752010Cleaning up temporary data from old regions at 1732131752024 (+14 ms)Region opened successfully at 1732131752032 (+8 ms)
2024-11-20T19:42:32,032 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:42:32,037 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3019ac31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:42:32,038 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:42:32,038 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:42:32,038 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:42:32,039 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:42:32,039 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:42:32,040 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:42:32,040 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:42:32,042 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:42:32,043 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:42:32,064 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:42:32,065 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:42:32,065 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:42:32,075 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:42:32,075 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:42:32,077 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:42:32,085 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:42:32,087 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:42:32,096 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:42:32,101 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:42:32,107 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:42:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:42:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,118 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,39803,1732131751605, sessionid=0x10159c502510000, setting cluster-up flag (Was=false)
2024-11-20T19:42:32,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,170 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:42:32,172 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,39803,1732131751605
2024-11-20T19:42:32,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,223 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:42:32,227 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,39803,1732131751605
2024-11-20T19:42:32,231 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:42:32,236 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:32,237 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:42:32,237 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:42:32,237 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,39803,1732131751605
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:42:32,239 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:32,239 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:42:32,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131782241
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:42:32,241 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:42:32,242 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:42:32,242 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,242 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:42:32,242 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:42:32,242 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:42:32,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:32,242 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:42:32,243 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131752243,5,FailOnTimeoutGroup]
2024-11-20T19:42:32,243 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131752243,5,FailOnTimeoutGroup]
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,243 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,244 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,244 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:42:32,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:42:32,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:42:32,254 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:42:32,254 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01
2024-11-20T19:42:32,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:42:32,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:42:32,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:32,262 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:42:32,264 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:42:32,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,264 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:32,264 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:42:32,266 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:42:32,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:32,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:42:32,268 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:42:32,268 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:32,269 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:42:32,270 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:42:32,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:32,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:32,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:42:32,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740
2024-11-20T19:42:32,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740
2024-11-20T19:42:32,274 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:42:32,274 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:42:32,274 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:42:32,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:42:32,278 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:42:32,279 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872198, jitterRate=0.1090577095746994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:42:32,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131752261Initializing all the Stores at 1732131752262 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131752262Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131752262Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131752262Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131752262Cleaning up temporary data from old regions at 1732131752274 (+12 ms)Region opened successfully at 1732131752303 (+29 ms)
2024-11-20T19:42:32,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:42:32,304 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:42:32,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:42:32,304 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(746): ClusterId : 3d7e43a1-686f-4331-b0fe-3e0c3371338b
2024-11-20T19:42:32,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:42:32,304 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:42:32,304 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:42:32,311 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:42:32,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131752304Disabling compacts and flushes for region at 1732131752304Disabling writes for close at 1732131752304Writing region close event to WAL at 1732131752311 (+7 ms)Closed at 1732131752311
2024-11-20T19:42:32,312 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:32,312 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:42:32,312 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:42:32,314 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:42:32,315 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:42:32,316 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:42:32,316 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:42:32,328 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:42:32,329 DEBUG [RS:0;e14ef07817f4:37815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d5ba6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:42:32,346 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:37815
2024-11-20T19:42:32,346 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:42:32,346 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:42:32,346 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:42:32,347 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,39803,1732131751605 with port=37815, startcode=1732131751777
2024-11-20T19:42:32,347 DEBUG [RS:0;e14ef07817f4:37815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:42:32,349 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47805, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:42:32,350 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,350 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39803 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,352 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01
2024-11-20T19:42:32,352 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37581
2024-11-20T19:42:32,352 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:42:32,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:42:32,413 DEBUG [RS:0;e14ef07817f4:37815 {}] zookeeper.ZKUtil(111): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,413 WARN  [RS:0;e14ef07817f4:37815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:42:32,413 INFO  [RS:0;e14ef07817f4:37815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:32,413 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,413 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,37815,1732131751777]
2024-11-20T19:42:32,418 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:42:32,421 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:42:32,422 INFO  [RS:0;e14ef07817f4:37815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:42:32,422 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,422 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:42:32,423 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:42:32,423 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,424 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,425 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:32,425 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:32,425 DEBUG [RS:0;e14ef07817f4:37815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,426 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,37815,1732131751777-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:42:32,442 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:42:32,442 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,37815,1732131751777-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,442 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,442 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.Replication(171): e14ef07817f4,37815,1732131751777 started
2024-11-20T19:42:32,454 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:32,454 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,37815,1732131751777, RpcServer on e14ef07817f4/172.17.0.3:37815, sessionid=0x10159c502510001
2024-11-20T19:42:32,454 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:42:32,454 DEBUG [RS:0;e14ef07817f4:37815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,454 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,37815,1732131751777'
2024-11-20T19:42:32,454 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,37815,1732131751777'
2024-11-20T19:42:32,455 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:42:32,456 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:42:32,456 DEBUG [RS:0;e14ef07817f4:37815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:42:32,456 INFO  [RS:0;e14ef07817f4:37815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:42:32,456 INFO  [RS:0;e14ef07817f4:37815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:42:32,465 WARN  [e14ef07817f4:39803 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:42:32,561 INFO  [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C37815%2C1732131751777, suffix=, logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777, archiveDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs, maxLogs=32
2024-11-20T19:42:32,563 INFO  [RS:0;e14ef07817f4:37815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131752562
2024-11-20T19:42:32,570 INFO  [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562
2024-11-20T19:42:32,575 DEBUG [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34365:34365),(127.0.0.1/127.0.0.1:45335:45335)]
2024-11-20T19:42:32,716 DEBUG [e14ef07817f4:39803 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:42:32,717 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:32,721 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,37815,1732131751777, state=OPENING
2024-11-20T19:42:32,748 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:42:32,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:42:32,836 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:42:32,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:32,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:32,837 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,37815,1732131751777}]
2024-11-20T19:42:32,994 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:42:32,999 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34197, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:42:33,006 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:42:33,007 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:33,011 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C37815%2C1732131751777.meta, suffix=.meta, logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777, archiveDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs, maxLogs=32
2024-11-20T19:42:33,012 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
2024-11-20T19:42:33,019 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
2024-11-20T19:42:33,020 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45335:45335),(127.0.0.1/127.0.0.1:34365:34365)]
2024-11-20T19:42:33,022 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:42:33,022 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:42:33,022 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:42:33,023 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:42:33,023 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:42:33,023 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:33,023 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:42:33,023 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:42:33,025 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:42:33,027 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:42:33,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,028 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:33,028 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:42:33,029 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:42:33,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,029 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:33,030 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:42:33,030 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:42:33,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,031 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:33,031 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:42:33,032 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:42:33,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,032 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:42:33,033 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:42:33,033 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740
2024-11-20T19:42:33,035 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740
2024-11-20T19:42:33,036 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:42:33,036 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:42:33,036 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:42:33,038 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:42:33,038 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817294, jitterRate=0.03924313187599182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:42:33,039 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:42:33,039 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131753023Writing region info on filesystem at 1732131753023Initializing all the Stores at 1732131753024 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131753025 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131753025Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131753025Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131753025Cleaning up temporary data from old regions at 1732131753036 (+11 ms)Running coprocessor post-open hooks at 1732131753039 (+3 ms)Region opened successfully at 1732131753039
2024-11-20T19:42:33,040 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131752994
2024-11-20T19:42:33,043 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:42:33,043 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:42:33,044 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:33,045 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,37815,1732131751777, state=OPEN
2024-11-20T19:42:33,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:42:33,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:42:33,083 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:33,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:33,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:42:33,091 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:42:33,091 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,37815,1732131751777 in 247 msec
2024-11-20T19:42:33,096 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:42:33,096 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec
2024-11-20T19:42:33,097 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:42:33,097 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:42:33,099 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:42:33,099 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,37815,1732131751777, seqNum=-1]
2024-11-20T19:42:33,099 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:42:33,101 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:42:33,107 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 872 msec
2024-11-20T19:42:33,107 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131753107, completionTime=-1
2024-11-20T19:42:33,107 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:42:33,107 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:42:33,109 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:42:33,109 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131813109
2024-11-20T19:42:33,109 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732131873109
2024-11-20T19:42:33,109 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-20T19:42:33,109 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,110 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,110 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,110 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:39803, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,110 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,110 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,112 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.270sec
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:42:33,114 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:42:33,117 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:42:33,117 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:42:33,117 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,39803,1732131751605-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794a0c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:33,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,39803,-1 for getting cluster id
2024-11-20T19:42:33,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:42:33,207 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3d7e43a1-686f-4331-b0fe-3e0c3371338b'
2024-11-20T19:42:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:42:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3d7e43a1-686f-4331-b0fe-3e0c3371338b"

2024-11-20T19:42:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6927e468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,39803,-1]
2024-11-20T19:42:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:42:33,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:42:33,212 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:42:33,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6702499c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:42:33,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:42:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,37815,1732131751777, seqNum=-1]
2024-11-20T19:42:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:42:33,218 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:42:33,221 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,39803,1732131751605
2024-11-20T19:42:33,221 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:33,225 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:42:33,242 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:42:33,243 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41391
2024-11-20T19:42:33,244 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41391 connecting to ZooKeeper ensemble=127.0.0.1:62155
2024-11-20T19:42:33,245 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:33,246 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:42:33,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413910x0, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:42:33,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41391-0x10159c502510002 connected
2024-11-20T19:42:33,265 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-20T19:42:33,265 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-20T19:42:33,266 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:42:33,268 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:42:33,269 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:42:33,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:42:33,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41391
2024-11-20T19:42:33,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41391
2024-11-20T19:42:33,273 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41391
2024-11-20T19:42:33,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41391
2024-11-20T19:42:33,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41391
2024-11-20T19:42:33,278 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(746): ClusterId : 3d7e43a1-686f-4331-b0fe-3e0c3371338b
2024-11-20T19:42:33,279 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:42:33,286 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:42:33,286 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:42:33,297 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:42:33,298 DEBUG [RS:1;e14ef07817f4:41391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9472a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:42:33,314 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e14ef07817f4:41391
2024-11-20T19:42:33,314 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:42:33,314 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:42:33,314 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:42:33,315 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,39803,1732131751605 with port=41391, startcode=1732131753241
2024-11-20T19:42:33,315 DEBUG [RS:1;e14ef07817f4:41391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:42:33,317 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47067, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:42:33,317 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,317 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39803 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,319 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01
2024-11-20T19:42:33,319 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37581
2024-11-20T19:42:33,319 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:42:33,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:42:33,328 DEBUG [RS:1;e14ef07817f4:41391 {}] zookeeper.ZKUtil(111): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,328 WARN  [RS:1;e14ef07817f4:41391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:42:33,328 INFO  [RS:1;e14ef07817f4:41391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:42:33,328 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,41391,1732131753241]
2024-11-20T19:42:33,328 DEBUG [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,332 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:42:33,334 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:42:33,335 INFO  [RS:1;e14ef07817f4:41391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:42:33,335 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,335 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:42:33,336 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:42:33,337 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,337 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,338 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,338 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,338 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:42:33,338 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:33,338 DEBUG [RS:1;e14ef07817f4:41391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:42:33,338 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,338 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,339 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,339 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,339 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,339 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,41391,1732131753241-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:42:33,354 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:42:33,354 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,41391,1732131753241-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,354 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,354 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.Replication(171): e14ef07817f4,41391,1732131753241 started
2024-11-20T19:42:33,366 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:42:33,366 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,41391,1732131753241, RpcServer on e14ef07817f4/172.17.0.3:41391, sessionid=0x10159c502510002
2024-11-20T19:42:33,366 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:42:33,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;e14ef07817f4:41391,5,FailOnTimeoutGroup]
2024-11-20T19:42:33,366 DEBUG [RS:1;e14ef07817f4:41391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,366 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,41391,1732131753241'
2024-11-20T19:42:33,367 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:42:33,367 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-20T19:42:33,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T19:42:33,367 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:42:33,368 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:42:33,368 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:42:33,368 DEBUG [RS:1;e14ef07817f4:41391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,41391,1732131753241
2024-11-20T19:42:33,368 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,41391,1732131753241'
2024-11-20T19:42:33,368 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:42:33,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is e14ef07817f4,39803,1732131751605
2024-11-20T19:42:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18f78161
2024-11-20T19:42:33,369 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:42:33,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T19:42:33,369 DEBUG [RS:1;e14ef07817f4:41391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:42:33,369 INFO  [RS:1;e14ef07817f4:41391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:42:33,369 INFO  [RS:1;e14ef07817f4:41391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:42:33,371 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T19:42:33,371 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T19:42:33,371 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T19:42:33,371 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:42:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T19:42:33,374 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T19:42:33,374 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,374 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-20T19:42:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:42:33,376 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T19:42:33,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741835_1011 (size=393)
2024-11-20T19:42:33,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741835_1011 (size=393)
2024-11-20T19:42:33,386 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dfa054c616ccab24a1fd25ac7e79b9ad, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01
2024-11-20T19:42:33,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32909 is added to blk_1073741836_1012 (size=76)
2024-11-20T19:42:33,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39335 is added to blk_1073741836_1012 (size=76)
2024-11-20T19:42:33,393 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:33,393 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing dfa054c616ccab24a1fd25ac7e79b9ad, disabling compactions & flushes
2024-11-20T19:42:33,393 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,394 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,394 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad. after waiting 0 ms
2024-11-20T19:42:33,394 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,394 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,394 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for dfa054c616ccab24a1fd25ac7e79b9ad:
Waiting for close lock at 1732131753393Disabling compacts and flushes for region at 1732131753393Disabling writes for close at 1732131753394 (+1 ms)Writing region close event to WAL at 1732131753394Closed at 1732131753394
2024-11-20T19:42:33,395 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T19:42:33,396 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732131753395"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131753395"}]},"ts":"1732131753395"}
2024-11-20T19:42:33,398 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T19:42:33,399 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T19:42:33,399 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131753399"}]},"ts":"1732131753399"}
2024-11-20T19:42:33,401 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-20T19:42:33,402 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dfa054c616ccab24a1fd25ac7e79b9ad, ASSIGN}]
2024-11-20T19:42:33,403 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dfa054c616ccab24a1fd25ac7e79b9ad, ASSIGN
2024-11-20T19:42:33,404 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dfa054c616ccab24a1fd25ac7e79b9ad, ASSIGN; state=OFFLINE, location=e14ef07817f4,37815,1732131751777; forceNewPlan=false, retain=false
2024-11-20T19:42:33,474 INFO  [RS:1;e14ef07817f4:41391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C41391%2C1732131753241, suffix=, logDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241, archiveDir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs, maxLogs=32
2024-11-20T19:42:33,476 INFO  [RS:1;e14ef07817f4:41391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41391%2C1732131753241.1732131753476
2024-11-20T19:42:33,487 INFO  [RS:1;e14ef07817f4:41391 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
2024-11-20T19:42:33,488 DEBUG [RS:1;e14ef07817f4:41391 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34365:34365),(127.0.0.1/127.0.0.1:45335:45335)]
2024-11-20T19:42:33,556 INFO  [e14ef07817f4:39803 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-20T19:42:33,556 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dfa054c616ccab24a1fd25ac7e79b9ad, regionState=OPENING, regionLocation=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:33,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dfa054c616ccab24a1fd25ac7e79b9ad, ASSIGN because future has completed
2024-11-20T19:42:33,563 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfa054c616ccab24a1fd25ac7e79b9ad, server=e14ef07817f4,37815,1732131751777}]
2024-11-20T19:42:33,730 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,731 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dfa054c616ccab24a1fd25ac7e79b9ad, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:42:33,732 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,732 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:42:33,733 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,733 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,735 INFO  [StoreOpener-dfa054c616ccab24a1fd25ac7e79b9ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dfa054c616ccab24a1fd25ac7e79b9ad 
2024-11-20T19:42:33,737 INFO  [StoreOpener-dfa054c616ccab24a1fd25ac7e79b9ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dfa054c616ccab24a1fd25ac7e79b9ad columnFamilyName info
2024-11-20T19:42:33,737 DEBUG [StoreOpener-dfa054c616ccab24a1fd25ac7e79b9ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:42:33,738 INFO  [StoreOpener-dfa054c616ccab24a1fd25ac7e79b9ad-1 {}] regionserver.HStore(327): Store=dfa054c616ccab24a1fd25ac7e79b9ad/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:42:33,738 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,739 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,740 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,740 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,740 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,743 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,746 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:42:33,746 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dfa054c616ccab24a1fd25ac7e79b9ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825007, jitterRate=0.0490516722202301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:42:33,746 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:33,747 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dfa054c616ccab24a1fd25ac7e79b9ad:
Running coprocessor pre-open hook at 1732131753733Writing region info on filesystem at 1732131753733Initializing all the Stores at 1732131753734 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131753734Cleaning up temporary data from old regions at 1732131753741 (+7 ms)Running coprocessor post-open hooks at 1732131753746 (+5 ms)Region opened successfully at 1732131753747 (+1 ms)
2024-11-20T19:42:33,748 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., pid=6, masterSystemTime=1732131753719
2024-11-20T19:42:33,750 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,750 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:33,751 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dfa054c616ccab24a1fd25ac7e79b9ad, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,37815,1732131751777
2024-11-20T19:42:33,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfa054c616ccab24a1fd25ac7e79b9ad, server=e14ef07817f4,37815,1732131751777 because future has completed
2024-11-20T19:42:33,759 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T19:42:33,759 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dfa054c616ccab24a1fd25ac7e79b9ad, server=e14ef07817f4,37815,1732131751777 in 193 msec
2024-11-20T19:42:33,761 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T19:42:33,762 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dfa054c616ccab24a1fd25ac7e79b9ad, ASSIGN in 357 msec
2024-11-20T19:42:33,763 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T19:42:33,763 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131753763"}]},"ts":"1732131753763"}
2024-11-20T19:42:33,765 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-20T19:42:33,766 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T19:42:33,769 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 394 msec
2024-11-20T19:42:34,024 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:42:34,026 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:34,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:34,046 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:34,046 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:37,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:42:37,373 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T19:42:37,377 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T19:42:37,377 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-20T19:42:37,379 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:42:37,379 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T19:42:38,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-20T19:42:39,238 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:42:39,244 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:39,262 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:39,265 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:39,265 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:42:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:42:43,399 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-20T19:42:43,399 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-20T19:42:43,406 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T19:42:43,406 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:43,420 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:43,424 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:43,424 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:43,424 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:43,424 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:42:43,425 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@65349436{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:43,425 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ab1ed71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:43,517 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c5f7ee0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-46423-hadoop-hdfs-3_4_1-tests_jar-_-any-8195846436713669886/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:43,518 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@495a6aea{HTTP/1.1, (http/1.1)}{localhost:46423}
2024-11-20T19:42:43,518 INFO  [Time-limited test {}] server.Server(415): Started @122592ms
2024-11-20T19:42:43,519 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:43,545 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:43,548 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:43,548 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:43,549 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:43,549 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:42:43,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@514acf4b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:43,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cde9b58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:43,643 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6079f97a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-36357-hadoop-hdfs-3_4_1-tests_jar-_-any-18315737369062969624/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:43,643 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1196c8fc{HTTP/1.1, (http/1.1)}{localhost:36357}
2024-11-20T19:42:43,643 INFO  [Time-limited test {}] server.Server(415): Started @122718ms
2024-11-20T19:42:43,645 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:43,676 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:43,679 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:43,680 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:43,680 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:43,680 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:42:43,681 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@136e75a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:43,681 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37673872{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:43,775 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b65aae1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-33705-hadoop-hdfs-3_4_1-tests_jar-_-any-18061430187136890270/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:43,776 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e08dd81{HTTP/1.1, (http/1.1)}{localhost:33705}
2024-11-20T19:42:43,776 INFO  [Time-limited test {}] server.Server(415): Started @122850ms
2024-11-20T19:42:43,777 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:44,954 WARN  [Thread-870 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data5/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:44,954 WARN  [Thread-871 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data6/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:44,975 WARN  [Thread-811 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:44,978 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96f32c84f6f59b4b with lease ID 0x32e92df50b0f620b: Processing first storage report for DS-2d3c775f-9560-4c03-a3a4-8777ae999808 from datanode DatanodeRegistration(127.0.0.1:41061, datanodeUuid=afc1434f-0850-40f4-aa51-2daacc675362, infoPort=38987, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:44,978 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96f32c84f6f59b4b with lease ID 0x32e92df50b0f620b: from storage DS-2d3c775f-9560-4c03-a3a4-8777ae999808 node DatanodeRegistration(127.0.0.1:41061, datanodeUuid=afc1434f-0850-40f4-aa51-2daacc675362, infoPort=38987, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:44,978 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96f32c84f6f59b4b with lease ID 0x32e92df50b0f620b: Processing first storage report for DS-d7c2403f-6361-40b8-b383-4a25c4739f6b from datanode DatanodeRegistration(127.0.0.1:41061, datanodeUuid=afc1434f-0850-40f4-aa51-2daacc675362, infoPort=38987, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:44,978 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96f32c84f6f59b4b with lease ID 0x32e92df50b0f620b: from storage DS-d7c2403f-6361-40b8-b383-4a25c4739f6b node DatanodeRegistration(127.0.0.1:41061, datanodeUuid=afc1434f-0850-40f4-aa51-2daacc675362, infoPort=38987, infoSecurePort=0, ipcPort=43745, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:45,378 WARN  [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data7/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:45,378 WARN  [Thread-882 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data8/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:45,394 WARN  [Thread-833 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:45,397 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x684a81b39b93188e with lease ID 0x32e92df50b0f620c: Processing first storage report for DS-db738247-a8f9-4806-bda5-3595072457be from datanode DatanodeRegistration(127.0.0.1:42847, datanodeUuid=f682965f-0390-446b-ac8c-24899a1b93e7, infoPort=42393, infoSecurePort=0, ipcPort=45797, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:45,397 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x684a81b39b93188e with lease ID 0x32e92df50b0f620c: from storage DS-db738247-a8f9-4806-bda5-3595072457be node DatanodeRegistration(127.0.0.1:42847, datanodeUuid=f682965f-0390-446b-ac8c-24899a1b93e7, infoPort=42393, infoSecurePort=0, ipcPort=45797, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:45,397 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x684a81b39b93188e with lease ID 0x32e92df50b0f620c: Processing first storage report for DS-3871b72a-a4da-4bd1-946a-fb03775b1132 from datanode DatanodeRegistration(127.0.0.1:42847, datanodeUuid=f682965f-0390-446b-ac8c-24899a1b93e7, infoPort=42393, infoSecurePort=0, ipcPort=45797, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:45,397 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x684a81b39b93188e with lease ID 0x32e92df50b0f620c: from storage DS-3871b72a-a4da-4bd1-946a-fb03775b1132 node DatanodeRegistration(127.0.0.1:42847, datanodeUuid=f682965f-0390-446b-ac8c-24899a1b93e7, infoPort=42393, infoSecurePort=0, ipcPort=45797, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:42:45,426 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:45,426 WARN  [Thread-893 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10/current/BP-854921811-172.17.0.3-1732131749074/current, will proceed with Du for space computation calculation, 
2024-11-20T19:42:45,445 WARN  [Thread-855 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:45,447 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8460bd22e8a9a0c8 with lease ID 0x32e92df50b0f620d: Processing first storage report for DS-6cd9a2f8-d893-4067-b039-a9f975723cfb from datanode DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:45,447 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8460bd22e8a9a0c8 with lease ID 0x32e92df50b0f620d: from storage DS-6cd9a2f8-d893-4067-b039-a9f975723cfb node DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:42:45,447 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8460bd22e8a9a0c8 with lease ID 0x32e92df50b0f620d: Processing first storage report for DS-d220e0dc-49bc-464f-b4f4-41102b64966d from datanode DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074)
2024-11-20T19:42:45,447 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8460bd22e8a9a0c8 with lease ID 0x32e92df50b0f620d: from storage DS-d220e0dc-49bc-464f-b4f4-41102b64966d node DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:45,514 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,514 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,514 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,514 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 block BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:45,514 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 block BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:45,514 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,515 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 block BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:45,515 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta block BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:45,515 WARN  [PacketResponder: BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39335] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,516 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48b0be64{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:45,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1662092964_22 at /127.0.0.1:49616 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49616 dst: /127.0.0.1:39335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-954799220_22 at /127.0.0.1:49670 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:39335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49670 dst: /127.0.0.1:39335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:54484 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:32909:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54484 dst: /127.0.0.1:32909
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,516 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4cae84f2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:45,517 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:45,517 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:54500 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:32909:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54500 dst: /127.0.0.1:32909
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,516 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:49642 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49642 dst: /127.0.0.1:39335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,517 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2dc8ddff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:45,517 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57ebe64c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:45,517 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1662092964_22 at /127.0.0.1:54456 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:32909:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54456 dst: /127.0.0.1:32909
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,517 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-954799220_22 at /127.0.0.1:54536 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:32909:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54536 dst: /127.0.0.1:32909
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:49648 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39335:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49648 dst: /127.0.0.1:39335
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:45,518 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:45,518 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:45,518 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid 501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9) service to localhost/127.0.0.1:37581
2024-11-20T19:42:45,518 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:45,519 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data4/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:45,519 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:45,519 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data3/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:45,523 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 block BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,523 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 block BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,523 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta block BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,523 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5afc739a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:45,523 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@200d0f88{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:45,523 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 block BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,524 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:45,524 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21f2acf7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:45,524 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@180ba686{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:45,525 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:45,525 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:45,525 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid 71812e8a-22fd-4421-a96c-a4d30768b529) service to localhost/127.0.0.1:37581
2024-11-20T19:42:45,525 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:45,526 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data1/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:45,526 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data2/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:45,526 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:45,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., hostname=e14ef07817f4,37815,1732131751777, seqNum=2]
2024-11-20T19:42:45,531 ERROR [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01-prefix:e14ef07817f4,37815,1732131751777 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,531 WARN  [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01-prefix:e14ef07817f4,37815,1732131751777 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,531 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,531 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C37815%2C1732131751777:(num 1732131752562) roll requested
2024-11-20T19:42:45,531 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131765531
2024-11-20T19:42:45,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:45,534 WARN  [Thread-903 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741838_1018
2024-11-20T19:42:45,536 WARN  [Thread-903 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:45,542 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:45,542 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:45,542 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:45,542 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:45,542 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:45,542 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531
2024-11-20T19:42:45,543 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,543 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:45,543 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38987:38987),(127.0.0.1/127.0.0.1:42393:42393)]
2024-11-20T19:42:45,543 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:45,544 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-20T19:42:45,544 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-20T19:42:45,544 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562
2024-11-20T19:42:45,547 WARN  [IPC Server handler 0 on default port 37581 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009
2024-11-20T19:42:45,550 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 after 4ms
2024-11-20T19:42:45,976 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:47,340 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:47,543 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:47,545 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531
2024-11-20T19:42:47,546 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:47,547 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531 block BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:47,548 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33660 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:41061:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33660 dst: /127.0.0.1:41061
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:47,548 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:49990 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:42847:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49990 dst: /127.0.0.1:42847
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:47,594 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c5f7ee0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:47,594 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@495a6aea{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:47,594 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:47,595 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ab1ed71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:47,595 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@65349436{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:47,596 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:47,596 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid afc1434f-0850-40f4-aa51-2daacc675362) service to localhost/127.0.0.1:37581
2024-11-20T19:42:47,596 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:47,597 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:47,597 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data5/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:47,597 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data6/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:47,597 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:47,977 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:49,340 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:49,545 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]
2024-11-20T19:42:49,545 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:49,546 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C37815%2C1732131751777:(num 1732131765531) roll requested
2024-11-20T19:42:49,547 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131769546
2024-11-20T19:42:49,552 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 after 4008ms
2024-11-20T19:42:49,555 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:49,555 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:49,555 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741840_1022
2024-11-20T19:42:49,556 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:49,560 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:49,561 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:49,561 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:49,561 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:49,561 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:49,561 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131769546
2024-11-20T19:42:49,562 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42393:42393),(127.0.0.1/127.0.0.1:43673:43673)]
2024-11-20T19:42:49,562 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:49,562 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531 is not closed yet, will try archiving it next time
2024-11-20T19:42:49,563 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42847 is added to blk_1073741839_1021 (size=2431)
2024-11-20T19:42:49,604 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T19:42:49,966 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:49,978 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,341 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,415 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5eac490c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42847, datanodeUuid=f682965f-0390-446b-ac8c-24899a1b93e7, infoPort=42393, infoSecurePort=0, ipcPort=45797, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741839_1021 to 127.0.0.1:39335 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,563 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,611 WARN  [ResponseProcessor for block BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,612 WARN  [DataStreamer for file /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131769546 block BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:51,613 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:50010 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:42847:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50010 dst: /127.0.0.1:42847
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,614 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41454 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41454 dst: /127.0.0.1:44123
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,647 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6079f97a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:51,647 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1196c8fc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:42:51,647 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:42:51,648 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cde9b58{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:42:51,648 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@514acf4b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:42:51,650 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:42:51,650 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:42:51,651 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:42:51,651 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid f682965f-0390-446b-ac8c-24899a1b93e7) service to localhost/127.0.0.1:37581
2024-11-20T19:42:51,652 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data7/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:51,653 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data8/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:42:51,653 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:42:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:51,663 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:42:51,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/cd5d34d5976c4242ae7b2ea32afc334f is 1080, key is row0002/info:/1732131767599/Put/seqid=0
2024-11-20T19:42:51,682 WARN  [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,683 WARN  [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:51,683 WARN  [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741842_1025
2024-11-20T19:42:51,683 WARN  [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:51,685 WARN  [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42847
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,685 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41466 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026 to mirror 127.0.0.1:42847
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,685 WARN  [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:51,685 WARN  [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026
2024-11-20T19:42:51,686 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41466 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:51,686 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41466 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741843_1026] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41466 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,686 WARN  [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:51,688 WARN  [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41061
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41482 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027 to mirror 127.0.0.1:41061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,688 WARN  [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:51,688 WARN  [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027
2024-11-20T19:42:51,688 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41482 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:51,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41482 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41482 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,689 WARN  [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:51,691 WARN  [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39335
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:51,691 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41492 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028 to mirror 127.0.0.1:39335
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,691 WARN  [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:51,691 WARN  [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028
2024-11-20T19:42:51,691 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41492 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:51,691 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:41492 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41492 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:51,692 WARN  [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:51,693 WARN  [IPC Server handler 4 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:51,693 WARN  [IPC Server handler 4 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:51,693 WARN  [IPC Server handler 4 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:51,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741846_1029 (size=10347)
2024-11-20T19:42:51,978 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:52,097 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/cd5d34d5976c4242ae7b2ea32afc334f
2024-11-20T19:42:52,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/cd5d34d5976c4242ae7b2ea32afc334f as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f
2024-11-20T19:42:52,115 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f, entries=5, sequenceid=11, filesize=10.1 K
2024-11-20T19:42:52,116 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for dfa054c616ccab24a1fd25ac7e79b9ad in 453ms, sequenceid=11, compaction requested=false
2024-11-20T19:42:52,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:52,294 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-20T19:42:52,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/23b8840b1f1348d78f1964f627ac1821 is 1080, key is row0007/info:/1732131771664/Put/seqid=0
2024-11-20T19:42:52,303 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:52,304 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:52,304 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741847_1030
2024-11-20T19:42:52,304 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:52,305 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:52,305 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:52,305 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741848_1031
2024-11-20T19:42:52,306 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:52,307 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:52,307 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:52,307 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741849_1032
2024-11-20T19:42:52,308 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:52,309 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:52,309 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:52,309 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741850_1033
2024-11-20T19:42:52,310 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:52,310 WARN  [IPC Server handler 4 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:52,310 WARN  [IPC Server handler 4 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:52,310 WARN  [IPC Server handler 4 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:52,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741851_1034 (size=12506)
2024-11-20T19:42:52,715 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/23b8840b1f1348d78f1964f627ac1821
2024-11-20T19:42:52,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/23b8840b1f1348d78f1964f627ac1821 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821
2024-11-20T19:42:52,737 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821, entries=7, sequenceid=24, filesize=12.2 K
2024-11-20T19:42:52,739 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for dfa054c616ccab24a1fd25ac7e79b9ad in 444ms, sequenceid=24, compaction requested=false
2024-11-20T19:42:52,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:52,739 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-20T19:42:52,739 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:52,739 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821 because midkey is the same as first or last row
2024-11-20T19:42:53,342 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,563 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]
2024-11-20T19:42:53,563 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,563 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C37815%2C1732131751777:(num 1732131769546) roll requested
2024-11-20T19:42:53,564 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131773563
2024-11-20T19:42:53,566 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,566 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:53,566 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741852_1035
2024-11-20T19:42:53,567 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:53,569 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42847
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,569 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:32990 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036 to mirror 127.0.0.1:42847
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,569 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:53,569 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036
2024-11-20T19:42:53,569 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:32990 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T19:42:53,569 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:32990 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:32990 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,570 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:53,571 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,571 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:53,571 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741854_1037
2024-11-20T19:42:53,572 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:53,573 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,573 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:53,573 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741855_1038
2024-11-20T19:42:53,574 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:53,574 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:53,575 WARN  [IPC Server handler 2 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:53,575 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:53,577 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:53,577 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:53,578 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:53,578 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:53,578 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:53,578 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131769546 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131773563
2024-11-20T19:42:53,579 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43673:43673)]
2024-11-20T19:42:53,579 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:53,579 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131769546 is not closed yet, will try archiving it next time
2024-11-20T19:42:53,579 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131765531 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C37815%2C1732131751777.1732131765531
2024-11-20T19:42:53,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741841_1024 (size=25992)
2024-11-20T19:42:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:53,733 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T19:42:53,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/46e7937d17ec475e89037c54813fade0 is 1079, key is tmprow/info:/1732131773730/Put/seqid=0
2024-11-20T19:42:53,743 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,743 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:53,744 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741857_1040
2024-11-20T19:42:53,744 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:53,747 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41061
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,747 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33014 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041 to mirror 127.0.0.1:41061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,747 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:53,747 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041
2024-11-20T19:42:53,747 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33014 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:53,747 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33014 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33014 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,748 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:53,749 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,749 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:53,749 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741859_1042
2024-11-20T19:42:53,750 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:53,753 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42847
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,753 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33016 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043 to mirror 127.0.0.1:42847
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,753 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:53,753 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043
2024-11-20T19:42:53,753 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33016 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:53,753 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33016 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33016 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:53,754 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:53,755 WARN  [IPC Server handler 1 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:53,755 WARN  [IPC Server handler 1 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:53,755 WARN  [IPC Server handler 1 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:53,758 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741861_1044 (size=6027)
2024-11-20T19:42:53,979 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:53,982 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:54,159 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/46e7937d17ec475e89037c54813fade0
2024-11-20T19:42:54,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/46e7937d17ec475e89037c54813fade0 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0
2024-11-20T19:42:54,173 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0, entries=1, sequenceid=34, filesize=5.9 K
2024-11-20T19:42:54,174 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dfa054c616ccab24a1fd25ac7e79b9ad in 442ms, sequenceid=34, compaction requested=true
2024-11-20T19:42:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-20T19:42:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821 because midkey is the same as first or last row
2024-11-20T19:42:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dfa054c616ccab24a1fd25ac7e79b9ad:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:42:54,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:54,175 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:42:54,176 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:42:54,176 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1541): dfa054c616ccab24a1fd25ac7e79b9ad/info is initiating minor compaction (all files)
2024-11-20T19:42:54,176 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dfa054c616ccab24a1fd25ac7e79b9ad/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:54,176 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0] into tmpdir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp, totalSize=28.2 K
2024-11-20T19:42:54,177 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5d34d5976c4242ae7b2ea32afc334f, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732131767599
2024-11-20T19:42:54,177 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b8840b1f1348d78f1964f627ac1821, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732131771664
2024-11-20T19:42:54,178 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e7937d17ec475e89037c54813fade0, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732131773730
2024-11-20T19:42:54,191 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dfa054c616ccab24a1fd25ac7e79b9ad#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:42:54,191 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/bfcd5618b626408e886aef1b04da9393 is 1080, key is row0002/info:/1732131767599/Put/seqid=0
2024-11-20T19:42:54,193 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:54,193 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:54,193 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741862_1045
2024-11-20T19:42:54,194 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:54,195 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:54,195 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:54,195 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741863_1046
2024-11-20T19:42:54,196 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:54,197 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:54,197 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:54,197 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741864_1047
2024-11-20T19:42:54,197 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:54,199 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39335
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:54,199 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33066 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048 to mirror 127.0.0.1:39335
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:54,200 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:54,200 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048
2024-11-20T19:42:54,200 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33066 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:54,200 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33066 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33066 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:54,200 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:54,201 WARN  [IPC Server handler 3 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:54,201 WARN  [IPC Server handler 3 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:54,201 WARN  [IPC Server handler 3 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:54,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741866_1049 (size=17994)
2024-11-20T19:42:54,452 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@39e0612d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741846_1029 to 127.0.0.1:39335 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:54,453 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@37279861[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741851_1034 to 127.0.0.1:32909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:54,617 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/bfcd5618b626408e886aef1b04da9393 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393
2024-11-20T19:42:54,624 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dfa054c616ccab24a1fd25ac7e79b9ad/info of dfa054c616ccab24a1fd25ac7e79b9ad into bfcd5618b626408e886aef1b04da9393(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:54,624 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., storeName=dfa054c616ccab24a1fd25ac7e79b9ad/info, priority=13, startTime=1732131774174; duration=0sec
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 because midkey is the same as first or last row
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 because midkey is the same as first or last row
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 because midkey is the same as first or last row
2024-11-20T19:42:54,624 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:54,625 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dfa054c616ccab24a1fd25ac7e79b9ad:info
2024-11-20T19:42:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:55,167 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T19:42:55,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/ef63e0c5dec24f3493a33eddbc2f94f1 is 1079, key is tmprow/info:/1732131775164/Put/seqid=0
2024-11-20T19:42:55,177 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,177 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:55,177 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741867_1050
2024-11-20T19:42:55,178 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:55,180 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39335
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,180 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33080 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051 to mirror 127.0.0.1:39335
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,180 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:55,180 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051
2024-11-20T19:42:55,180 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33080 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:55,180 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33080 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33080 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,181 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:55,183 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41061
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33094 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052 to mirror 127.0.0.1:41061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,183 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:55,183 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33094 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:55,183 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052
2024-11-20T19:42:55,183 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33094 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33094 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,184 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:55,186 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33106 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053 to mirror 127.0.0.1:32909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,186 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,187 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33106 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:55,187 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:55,187 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053
2024-11-20T19:42:55,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33106 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33106 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,188 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:55,188 WARN  [IPC Server handler 0 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:55,188 WARN  [IPC Server handler 0 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:55,188 WARN  [IPC Server handler 0 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:55,191 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741871_1054 (size=6027)
2024-11-20T19:42:55,342 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,453 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@37279861[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741841_1024 to 127.0.0.1:32909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,453 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@39e0612d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741861_1044 to 127.0.0.1:41061 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,579 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]
2024-11-20T19:42:55,580 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,580 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C37815%2C1732131751777:(num 1732131773563) roll requested
2024-11-20T19:42:55,580 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131775580
2024-11-20T19:42:55,584 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,585 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:55,585 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741872_1055
2024-11-20T19:42:55,586 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:55,588 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,588 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:55,588 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741873_1056
2024-11-20T19:42:55,589 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:55,593 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/ef63e0c5dec24f3493a33eddbc2f94f1
2024-11-20T19:42:55,592 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41061
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33136 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057 to mirror 127.0.0.1:41061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,593 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:55,593 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057
2024-11-20T19:42:55,593 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33136 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T19:42:55,593 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33136 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33136 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,594 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:55,597 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:55,597 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:55,597 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33152 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058 to mirror 127.0.0.1:32909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,597 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058
2024-11-20T19:42:55,597 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33152 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T19:42:55,597 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33152 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33152 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:55,598 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:55,599 WARN  [IPC Server handler 0 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:55,599 WARN  [IPC Server handler 0 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:55,599 WARN  [IPC Server handler 0 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:55,601 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:55,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/ef63e0c5dec24f3493a33eddbc2f94f1 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1
2024-11-20T19:42:55,602 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:55,602 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:55,602 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:55,602 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:42:55,602 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131773563 with entries=14, filesize=12.92 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131775580
2024-11-20T19:42:55,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741856_1039 (size=13234)
2024-11-20T19:42:55,607 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1, entries=1, sequenceid=45, filesize=5.9 K
2024-11-20T19:42:55,609 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43673:43673)]
2024-11-20T19:42:55,609 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:55,609 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131773563 is not closed yet, will try archiving it next time
2024-11-20T19:42:55,609 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131769546 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C37815%2C1732131751777.1732131769546
2024-11-20T19:42:55,610 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dfa054c616ccab24a1fd25ac7e79b9ad in 443ms, sequenceid=45, compaction requested=false
2024-11-20T19:42:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-20T19:42:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:55,610 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 because midkey is the same as first or last row
2024-11-20T19:42:55,980 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:56,005 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 is not closed yet, will try archiving it next time
2024-11-20T19:42:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:42:56,604 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T19:42:56,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/2a766e6be8df4bf6aa7b1e641d3d5d05 is 1079, key is tmprow/info:/1732131776602/Put/seqid=0
2024-11-20T19:42:56,615 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:56,615 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:56,615 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741877_1060
2024-11-20T19:42:56,616 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:56,617 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:56,618 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:56,618 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741878_1061
2024-11-20T19:42:56,618 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:56,621 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42847
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:56,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33160 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062 to mirror 127.0.0.1:42847
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:56,621 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:56,621 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062
2024-11-20T19:42:56,621 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33160 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:56,621 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33160 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33160 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:56,622 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:56,625 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:32909
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:56,625 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33164 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063 to mirror 127.0.0.1:32909
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:56,625 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:56,625 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063
2024-11-20T19:42:56,626 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33164 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:42:56,626 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:33164 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33164 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:56,626 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:56,627 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:56,627 WARN  [IPC Server handler 2 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:56,627 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:56,630 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741881_1064 (size=6027)
2024-11-20T19:42:57,031 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/2a766e6be8df4bf6aa7b1e641d3d5d05
2024-11-20T19:42:57,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/2a766e6be8df4bf6aa7b1e641d3d5d05 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05
2024-11-20T19:42:57,050 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05, entries=1, sequenceid=55, filesize=5.9 K
2024-11-20T19:42:57,051 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dfa054c616ccab24a1fd25ac7e79b9ad in 447ms, sequenceid=55, compaction requested=true
2024-11-20T19:42:57,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-20T19:42:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 because midkey is the same as first or last row
2024-11-20T19:42:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dfa054c616ccab24a1fd25ac7e79b9ad:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:42:57,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:57,052 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:42:57,053 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:42:57,054 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1541): dfa054c616ccab24a1fd25ac7e79b9ad/info is initiating minor compaction (all files)
2024-11-20T19:42:57,054 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dfa054c616ccab24a1fd25ac7e79b9ad/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:42:57,054 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05] into tmpdir=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp, totalSize=29.3 K
2024-11-20T19:42:57,054 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfcd5618b626408e886aef1b04da9393, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732131767599
2024-11-20T19:42:57,055 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef63e0c5dec24f3493a33eddbc2f94f1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732131775164
2024-11-20T19:42:57,055 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a766e6be8df4bf6aa7b1e641d3d5d05, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732131776602
2024-11-20T19:42:57,073 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dfa054c616ccab24a1fd25ac7e79b9ad#info#compaction#24 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:42:57,073 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/abfc0de9c86c48baae7505e46e897b0c is 1080, key is row0002/info:/1732131767599/Put/seqid=0
2024-11-20T19:42:57,075 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,075 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:42:57,075 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741882_1065
2024-11-20T19:42:57,076 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:42:57,077 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,077 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:42:57,077 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741883_1066
2024-11-20T19:42:57,077 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:42:57,078 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,079 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]) is bad.
2024-11-20T19:42:57,079 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741884_1067
2024-11-20T19:42:57,079 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39335,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]
2024-11-20T19:42:57,080 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,080 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:42:57,080 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741885_1068
2024-11-20T19:42:57,081 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:42:57,082 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T19:42:57,082 WARN  [IPC Server handler 2 on default port 37581 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T19:42:57,082 WARN  [IPC Server handler 2 on default port 37581 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T19:42:57,085 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741886_1069 (size=18097)
2024-11-20T19:42:57,343 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,454 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@39e0612d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741866_1049 to 127.0.0.1:32909 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:57,454 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@37279861[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741871_1054 to 127.0.0.1:42847 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:57,501 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/abfc0de9c86c48baae7505e46e897b0c as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/abfc0de9c86c48baae7505e46e897b0c
2024-11-20T19:42:57,509 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dfa054c616ccab24a1fd25ac7e79b9ad/info of dfa054c616ccab24a1fd25ac7e79b9ad into abfc0de9c86c48baae7505e46e897b0c(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:42:57,509 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:42:57,509 INFO  [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., storeName=dfa054c616ccab24a1fd25ac7e79b9ad/info, priority=13, startTime=1732131777052; duration=0sec
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/abfc0de9c86c48baae7505e46e897b0c because midkey is the same as first or last row
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/abfc0de9c86c48baae7505e46e897b0c because midkey is the same as first or last row
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/abfc0de9c86c48baae7505e46e897b0c because midkey is the same as first or last row
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:42:57,510 DEBUG [RS:0;e14ef07817f4:37815-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dfa054c616ccab24a1fd25ac7e79b9ad:info
2024-11-20T19:42:57,610 WARN  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-20T19:42:57,610 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:57,637 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:42:57,641 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:42:57,642 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:42:57,642 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:42:57,642 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:42:57,642 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2586080f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:42:57,643 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c8ca2dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:42:57,736 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f32fff4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/java.io.tmpdir/jetty-localhost-36791-hadoop-hdfs-3_4_1-tests_jar-_-any-17493509506266578780/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:42:57,736 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@434810ac{HTTP/1.1, (http/1.1)}{localhost:36791}
2024-11-20T19:42:57,736 INFO  [Time-limited test {}] server.Server(415): Started @136810ms
2024-11-20T19:42:57,737 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:42:57,981 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:58,161 WARN  [Thread-993 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:42:58,168 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9d512e41edec74a with lease ID 0x32e92df50b0f620e: from storage DS-00acf580-b45e-483d-a65c-06dbb9accd8f node DatanodeRegistration(127.0.0.1:36869, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=37737, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:42:58,169 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9d512e41edec74a with lease ID 0x32e92df50b0f620e: from storage DS-c9386add-398d-4307-8b7a-362613faee15 node DatanodeRegistration(127.0.0.1:36869, datanodeUuid=501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9, infoPort=37737, infoSecurePort=0, ipcPort=34951, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:42:58,451 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@39e0612d[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741881_1064 to 127.0.0.1:41061 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:42:58,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741856_1039 (size=13234)
2024-11-20T19:42:59,343 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:59,610 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:42:59,981 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:00,452 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@37279861[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741886_1069 to 127.0.0.1:41061 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:01,344 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:01,586 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:43:01,611 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:01,982 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,243 ERROR [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData-prefix:e14ef07817f4,39803,1732131751605 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,243 WARN  [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData-prefix:e14ef07817f4,39803,1732131751605 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,244 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C39803%2C1732131751605:(num 1732131751999) roll requested
2024-11-20T19:43:02,244 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C39803%2C1732131751605.1732131782244
2024-11-20T19:43:02,249 WARN  [Thread-1014 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,250 WARN  [Thread-1014 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:43:02,250 WARN  [Thread-1014 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741887_1070
2024-11-20T19:43:02,251 WARN  [Thread-1014 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:43:02,257 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:02,257 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:02,258 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:02,258 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:02,258 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:02,258 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131782244
2024-11-20T19:43:02,259 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,259 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:02,259 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999
2024-11-20T19:43:02,260 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43673:43673),(127.0.0.1/127.0.0.1:37737:37737)]
2024-11-20T19:43:02,260 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 is not closed yet, will try archiving it next time
2024-11-20T19:43:02,260 WARN  [IPC Server handler 1 on default port 37581 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006
2024-11-20T19:43:02,260 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 after 1ms
2024-11-20T19:43:03,344 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:03,611 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:05,345 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:05,612 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:06,263 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 after 4004ms
2024-11-20T19:43:07,345 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:07,612 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:08,184 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4a8289f2 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-854921811-172.17.0.3-1732131749074:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:32909,null,null])
java.net.ConnectException: Call From e14ef07817f4/172.17.0.3 to localhost:46447 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T19:43:08,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741833_1020 (size=455)
2024-11-20T19:43:08,580 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131752562 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C37815%2C1732131751777.1732131752562
2024-11-20T19:43:08,584 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131773563 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C37815%2C1732131751777.1732131773563
2024-11-20T19:43:09,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741833_1020 (size=455)
2024-11-20T19:43:09,346 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:09,613 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,347 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,457 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.1732131791457
2024-11-20T19:43:11,463 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,464 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:36869,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:43:11,464 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741889_1073
2024-11-20T19:43:11,465 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:43:11,467 WARN  [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,468 WARN  [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:43:11,468 WARN  [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741890_1074
2024-11-20T19:43:11,469 WARN  [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:43:11,475 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,475 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,475 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,475 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,475 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,475 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131775580 with entries=13, filesize=11.81 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131791457
2024-11-20T19:43:11,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37737:37737),(127.0.0.1/127.0.0.1:43673:43673)]
2024-11-20T19:43:11,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131775580 is not closed yet, will try archiving it next time
2024-11-20T19:43:11,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741876_1059 (size=12100)
2024-11-20T19:43:11,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37815 {}] regionserver.HRegion(8855): Flush requested on dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:43:11,482 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T19:43:11,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/33633fcc214c44938050306b917bd59e is 1080, key is row0013/info:/1732131791477/Put/seqid=0
2024-11-20T19:43:11,490 WARN  [Thread-1031 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,490 WARN  [Thread-1031 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:43:11,490 WARN  [Thread-1031 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741892_1076
2024-11-20T19:43:11,491 WARN  [Thread-1031 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:43:11,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741893_1077 (size=9267)
2024-11-20T19:43:11,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741893_1077 (size=9267)
2024-11-20T19:43:11,501 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/33633fcc214c44938050306b917bd59e
2024-11-20T19:43:11,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/33633fcc214c44938050306b917bd59e as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/33633fcc214c44938050306b917bd59e
2024-11-20T19:43:11,514 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/33633fcc214c44938050306b917bd59e, entries=4, sequenceid=66, filesize=9.0 K
2024-11-20T19:43:11,515 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for dfa054c616ccab24a1fd25ac7e79b9ad in 33ms, sequenceid=66, compaction requested=false
2024-11-20T19:43:11,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dfa054c616ccab24a1fd25ac7e79b9ad:

2024-11-20T19:43:11,515 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=26.7 K, sizeToCheck=16.0 K
2024-11-20T19:43:11,516 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:43:11,516 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/abfc0de9c86c48baae7505e46e897b0c because midkey is the same as first or last row
2024-11-20T19:43:11,613 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-20T19:43:11,614 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,703 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:43:11,703 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:43:11,704 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:11,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:11,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:11,705 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:43:11,705 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:43:11,706 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1352773276, stopped=false
2024-11-20T19:43:11,706 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,39803,1732131751605
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:11,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:11,768 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:43:11,769 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:43:11,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:11,770 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:11,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:11,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:11,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:11,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,37815,1732131751777' *****
2024-11-20T19:43:11,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:43:11,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,41391,1732131753241' *****
2024-11-20T19:43:11,771 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:43:11,771 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:43:11,771 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:43:11,771 INFO  [RS:1;e14ef07817f4:41391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:43:11,772 INFO  [RS:0;e14ef07817f4:37815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:43:11,772 INFO  [RS:1;e14ef07817f4:41391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:43:11,772 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:43:11,772 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,41391,1732131753241
2024-11-20T19:43:11,772 INFO  [RS:0;e14ef07817f4:37815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:43:11,772 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:43:11,772 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(3091): Received CLOSE for dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:43:11,772 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:43:11,772 INFO  [RS:1;e14ef07817f4:41391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e14ef07817f4:41391.
2024-11-20T19:43:11,773 DEBUG [RS:1;e14ef07817f4:41391 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:11,773 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,37815,1732131751777
2024-11-20T19:43:11,773 DEBUG [RS:1;e14ef07817f4:41391 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:11,773 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:43:11,773 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,41391,1732131753241; all regions closed.
2024-11-20T19:43:11,773 INFO  [RS:0;e14ef07817f4:37815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:37815.
2024-11-20T19:43:11,773 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dfa054c616ccab24a1fd25ac7e79b9ad, disabling compactions & flushes
2024-11-20T19:43:11,773 DEBUG [RS:0;e14ef07817f4:37815 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:11,773 DEBUG [RS:0;e14ef07817f4:37815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:11,773 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:43:11,773 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:43:11,774 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:43:11,774 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:43:11,774 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad. after waiting 0 ms
2024-11-20T19:43:11,774 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:43:11,774 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:43:11,774 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:43:11,774 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,774 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,774 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dfa054c616ccab24a1fd25ac7e79b9ad 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-20T19:43:11,774 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,774 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T19:43:11,774 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,774 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1325): Online Regions={dfa054c616ccab24a1fd25ac7e79b9ad=TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:43:11,774 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,774 DEBUG [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dfa054c616ccab24a1fd25ac7e79b9ad
2024-11-20T19:43:11,775 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:43:11,775 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:43:11,775 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:43:11,775 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:43:11,775 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,775 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:43:11,775 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,775 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-20T19:43:11,775 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
2024-11-20T19:43:11,776 ERROR [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01-prefix:e14ef07817f4,37815,1732131751777.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,776 WARN  [FSHLog-0-hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01-prefix:e14ef07817f4,37815,1732131751777.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,776 WARN  [IPC Server handler 4 on default port 37581 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 has not been closed. Lease recovery is in progress. RecoveryId = 1078 for block blk_1073741837_1013
2024-11-20T19:43:11,776 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C37815%2C1732131751777.meta:.meta(num 1732131753012) roll requested
2024-11-20T19:43:11,777 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 after 2ms
2024-11-20T19:43:11,777 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C37815%2C1732131751777.meta.1732131791777.meta
2024-11-20T19:43:11,780 WARN  [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,780 WARN  [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741894_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:36869,DS-00acf580-b45e-483d-a65c-06dbb9accd8f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:43:11,780 WARN  [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741894_1079
2024-11-20T19:43:11,780 WARN  [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:43:11,781 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/79c93bf84d534b0592f3eb4e97d538e4 is 1080, key is row0016/info:/1732131791484/Put/seqid=0
2024-11-20T19:43:11,782 WARN  [Thread-1040 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,782 WARN  [Thread-1040 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741896_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:43:11,782 WARN  [Thread-1040 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741896_1081
2024-11-20T19:43:11,783 WARN  [Thread-1040 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:43:11,784 WARN  [Thread-1040 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1082
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,785 WARN  [Thread-1040 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741897_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:43:11,785 WARN  [Thread-1040 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741897_1082
2024-11-20T19:43:11,785 WARN  [Thread-1040 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:43:11,789 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,789 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,789 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,790 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,790 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,790 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131791777.meta
2024-11-20T19:43:11,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741898_1083 (size=13583)
2024-11-20T19:43:11,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741898_1083 (size=13583)
2024-11-20T19:43:11,792 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,792 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,792 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
2024-11-20T19:43:11,792 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/79c93bf84d534b0592f3eb4e97d538e4
2024-11-20T19:43:11,793 WARN  [IPC Server handler 1 on default port 37581 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta has not been closed. Lease recovery is in progress. RecoveryId = 1084 for block blk_1073741834_1010
2024-11-20T19:43:11,793 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta after 1ms
2024-11-20T19:43:11,793 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43673:43673),(127.0.0.1/127.0.0.1:37737:37737)]
2024-11-20T19:43:11,793 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta is not closed yet, will try archiving it next time
2024-11-20T19:43:11,799 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/.tmp/info/79c93bf84d534b0592f3eb4e97d538e4 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/79c93bf84d534b0592f3eb4e97d538e4
2024-11-20T19:43:11,805 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/79c93bf84d534b0592f3eb4e97d538e4, entries=8, sequenceid=77, filesize=13.3 K
2024-11-20T19:43:11,807 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for dfa054c616ccab24a1fd25ac7e79b9ad in 32ms, sequenceid=77, compaction requested=true
2024-11-20T19:43:11,807 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05] to archive
2024-11-20T19:43:11,808 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:43:11,811 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/cd5d34d5976c4242ae7b2ea32afc334f
2024-11-20T19:43:11,812 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/23b8840b1f1348d78f1964f627ac1821
2024-11-20T19:43:11,814 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/bfcd5618b626408e886aef1b04da9393
2024-11-20T19:43:11,816 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/46e7937d17ec475e89037c54813fade0
2024-11-20T19:43:11,817 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/info/23810b55e3e24865b96ea464ba653cf7 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad./info:regioninfo/1732131753751/Put/seqid=0
2024-11-20T19:43:11,818 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/ef63e0c5dec24f3493a33eddbc2f94f1
2024-11-20T19:43:11,820 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/info/2a766e6be8df4bf6aa7b1e641d3d5d05
2024-11-20T19:43:11,820 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e14ef07817f4:39803 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-20T19:43:11,820 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [cd5d34d5976c4242ae7b2ea32afc334f=10347, 23b8840b1f1348d78f1964f627ac1821=12506, bfcd5618b626408e886aef1b04da9393=17994, 46e7937d17ec475e89037c54813fade0=6027, ef63e0c5dec24f3493a33eddbc2f94f1=6027, 2a766e6be8df4bf6aa7b1e641d3d5d05=6027]
2024-11-20T19:43:11,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741899_1085 (size=7089)
2024-11-20T19:43:11,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741899_1085 (size=7089)
2024-11-20T19:43:11,823 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/info/23810b55e3e24865b96ea464ba653cf7
2024-11-20T19:43:11,825 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dfa054c616ccab24a1fd25ac7e79b9ad/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1
2024-11-20T19:43:11,826 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:43:11,826 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dfa054c616ccab24a1fd25ac7e79b9ad:
Waiting for close lock at 1732131791773Running coprocessor pre-close hooks at 1732131791773Disabling compacts and flushes for region at 1732131791773Disabling writes for close at 1732131791774 (+1 ms)Obtaining lock to block concurrent updates at 1732131791774Preparing flush snapshotting stores in dfa054c616ccab24a1fd25ac7e79b9ad at 1732131791774Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad., syncing WAL and waiting on mvcc, flushsize=dataSize=8607, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1732131791775 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad. at 1732131791776 (+1 ms)Flushing dfa054c616ccab24a1fd25ac7e79b9ad/info: creating writer at 1732131791776Flushing dfa054c616ccab24a1fd25ac7e79b9ad/info: appending metadata at 1732131791780 (+4 ms)Flushing dfa054c616ccab24a1fd25ac7e79b9ad/info: closing flushed file at 1732131791780Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1372eecc: reopening flushed file at 1732131791799 (+19 ms)Finished flush of dataSize ~8.41 KB/8607, heapSize ~9.23 KB/9456, currentSize=0 B/0 for dfa054c616ccab24a1fd25ac7e79b9ad in 32ms, sequenceid=77, compaction requested=true at 1732131791807 (+8 ms)Writing region close event to WAL at 1732131791821 (+14 ms)Running coprocessor post-close hooks at 1732131791826 (+5 ms)Closed at 1732131791826
2024-11-20T19:43:11,826 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732131753371.dfa054c616ccab24a1fd25ac7e79b9ad.
2024-11-20T19:43:11,849 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/ns/00070bec2424468884af0ac18555ff23 is 43, key is default/ns:d/1732131753101/Put/seqid=0
2024-11-20T19:43:11,851 WARN  [Thread-1058 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,851 WARN  [Thread-1058 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]) is bad.
2024-11-20T19:43:11,851 WARN  [Thread-1058 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741900_1086
2024-11-20T19:43:11,852 WARN  [Thread-1058 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:32909,DS-8f75b82d-0f90-4e6b-9f0d-c26e694e1707,DISK]
2024-11-20T19:43:11,853 WARN  [Thread-1058 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1087
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,853 WARN  [Thread-1058 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741901_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK], DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]) is bad.
2024-11-20T19:43:11,853 WARN  [Thread-1058 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741901_1087
2024-11-20T19:43:11,854 WARN  [Thread-1058 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42847,DS-db738247-a8f9-4806-bda5-3595072457be,DISK]
2024-11-20T19:43:11,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741902_1088 (size=5153)
2024-11-20T19:43:11,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741902_1088 (size=5153)
2024-11-20T19:43:11,860 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/ns/00070bec2424468884af0ac18555ff23
2024-11-20T19:43:11,878 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.1732131775580 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C37815%2C1732131751777.1732131775580
2024-11-20T19:43:11,884 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/table/e9bb39f8494541e691ee5790df2c165d is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732131753763/Put/seqid=0
2024-11-20T19:43:11,887 WARN  [Thread-1064 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41061
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:11,887 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:42962 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10]'}, localName='127.0.0.1:44123', datanodeUuid='eb7ba641-5546-4742-a765-e19409ef6022', xmitsInProgress=0}:Exception transferring block BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089 to mirror 127.0.0.1:41061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:11,887 WARN  [Thread-1064 {}] hdfs.DataStreamer(1731): Error Recovery for BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44123,DS-6cd9a2f8-d893-4067-b039-a9f975723cfb,DISK], DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]) is bad.
2024-11-20T19:43:11,887 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:42962 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089] {}] datanode.BlockReceiver(316): Block 1073741903 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T19:43:11,887 WARN  [Thread-1064 {}] hdfs.DataStreamer(1850): Abandoning BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089
2024-11-20T19:43:11,887 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1920600184_22 at /127.0.0.1:42962 [Receiving block BP-854921811-172.17.0.3-1732131749074:blk_1073741903_1089] {}] datanode.DataXceiver(331): 127.0.0.1:44123:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42962 dst: /127.0.0.1:44123
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:11,888 WARN  [Thread-1064 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41061,DS-2d3c775f-9560-4c03-a3a4-8777ae999808,DISK]
2024-11-20T19:43:11,892 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741904_1090 (size=5424)
2024-11-20T19:43:11,893 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741904_1090 (size=5424)
2024-11-20T19:43:11,893 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/table/e9bb39f8494541e691ee5790df2c165d
2024-11-20T19:43:11,901 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/info/23810b55e3e24865b96ea464ba653cf7 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/info/23810b55e3e24865b96ea464ba653cf7
2024-11-20T19:43:11,908 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/info/23810b55e3e24865b96ea464ba653cf7, entries=10, sequenceid=11, filesize=6.9 K
2024-11-20T19:43:11,909 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/ns/00070bec2424468884af0ac18555ff23 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/ns/00070bec2424468884af0ac18555ff23
2024-11-20T19:43:11,916 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/ns/00070bec2424468884af0ac18555ff23, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T19:43:11,917 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/.tmp/table/e9bb39f8494541e691ee5790df2c165d as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/table/e9bb39f8494541e691ee5790df2c165d
2024-11-20T19:43:11,924 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/table/e9bb39f8494541e691ee5790df2c165d, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T19:43:11,925 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 150ms, sequenceid=11, compaction requested=false
2024-11-20T19:43:11,931 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T19:43:11,931 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:43:11,931 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:11,932 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131791775Running coprocessor pre-close hooks at 1732131791775Disabling compacts and flushes for region at 1732131791775Disabling writes for close at 1732131791775Obtaining lock to block concurrent updates at 1732131791775Preparing flush snapshotting stores in 1588230740 at 1732131791775Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732131791776 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732131791794 (+18 ms)Flushing 1588230740/info: creating writer at 1732131791794Flushing 1588230740/info: appending metadata at 1732131791816 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732131791816Flushing 1588230740/ns: creating writer at 1732131791829 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732131791849 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732131791849Flushing 1588230740/table: creating writer at 1732131791867 (+18 ms)Flushing 1588230740/table: appending metadata at 1732131791883 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732131791883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@411797be: reopening flushed file at 1732131791900 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e268b5c: reopening flushed file at 1732131791908 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49de5c60: reopening flushed file at 1732131791916 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 150ms, sequenceid=11, compaction requested=false at 1732131791925 (+9 ms)Writing region close event to WAL at 1732131791927 (+2 ms)Running coprocessor post-close hooks at 1732131791931 (+4 ms)Closed at 1732131791931
2024-11-20T19:43:11,932 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:11,975 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,37815,1732131751777; all regions closed.
2024-11-20T19:43:11,975 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,975 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,976 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,976 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,976 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:11,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741895_1080 (size=825)
2024-11-20T19:43:11,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741895_1080 (size=825)
2024-11-20T19:43:12,380 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T19:43:12,380 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T19:43:12,426 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T19:43:12,426 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T19:43:12,429 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:12,454 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@37279861[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44123, datanodeUuid=eb7ba641-5546-4742-a765-e19409ef6022, infoPort=43673, infoSecurePort=0, ipcPort=44491, storageInfo=lv=-57;cid=testClusterID;nsid=113537729;c=1732131749074):Failed to transfer BP-854921811-172.17.0.3-1732131749074:blk_1073741876_1059 to 127.0.0.1:41061 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:13,118 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T19:43:13,118 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T19:43:13,342 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:15,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741835_1011 (size=393)
2024-11-20T19:43:15,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:43:15,779 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 after 4004ms
2024-11-20T19:43:15,812 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta after 4019ms
2024-11-20T19:43:16,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:43:16,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:43:16,776 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-20T19:43:16,782 DEBUG [RS:1;e14ef07817f4:41391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs
2024-11-20T19:43:16,782 INFO  [RS:1;e14ef07817f4:41391 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C41391%2C1732131753241:(num 1732131753476)
2024-11-20T19:43:16,782 DEBUG [RS:1;e14ef07817f4:41391 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:16,782 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:16,783 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:43:16,783 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:43:16,784 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:43:16,784 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:43:16,784 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:43:16,784 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:43:16,784 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:43:16,784 INFO  [RS:1;e14ef07817f4:41391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41391
2024-11-20T19:43:16,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,849 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,849 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,850 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,865 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,865 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:16,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,41391,1732131753241
2024-11-20T19:43:16,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:43:16,902 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:43:16,912 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,41391,1732131753241]
2024-11-20T19:43:16,923 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,41391,1732131753241 already deleted, retry=false
2024-11-20T19:43:16,923 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,41391,1732131753241 expired; onlineServers=1
2024-11-20T19:43:16,976 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-20T19:43:16,983 DEBUG [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs
2024-11-20T19:43:16,983 INFO  [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C37815%2C1732131751777.meta:.meta(num 1732131791777)
2024-11-20T19:43:16,984 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:16,985 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:16,985 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:16,985 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:16,985 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:16,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741891_1075 (size=14682)
2024-11-20T19:43:16,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741891_1075 (size=14682)
2024-11-20T19:43:16,991 DEBUG [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs
2024-11-20T19:43:16,991 INFO  [RS:0;e14ef07817f4:37815 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C37815%2C1732131751777:(num 1732131791457)
2024-11-20T19:43:16,991 DEBUG [RS:0;e14ef07817f4:37815 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:16,991 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:16,991 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:43:16,992 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:43:16,992 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:43:16,992 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:43:16,992 INFO  [RS:0;e14ef07817f4:37815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37815
2024-11-20T19:43:16,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,37815,1732131751777
2024-11-20T19:43:16,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:43:16,999 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:43:17,010 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,37815,1732131751777]
2024-11-20T19:43:17,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:17,012 INFO  [RS:1;e14ef07817f4:41391 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:43:17,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41391-0x10159c502510002, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:17,012 INFO  [RS:1;e14ef07817f4:41391 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,41391,1732131753241; zookeeper connection closed.
2024-11-20T19:43:17,013 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49ac6b3b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49ac6b3b
2024-11-20T19:43:17,020 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,37815,1732131751777 already deleted, retry=false
2024-11-20T19:43:17,020 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,37815,1732131751777 expired; onlineServers=0
2024-11-20T19:43:17,020 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,39803,1732131751605' *****
2024-11-20T19:43:17,020 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:43:17,021 INFO  [M:0;e14ef07817f4:39803 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:43:17,021 INFO  [M:0;e14ef07817f4:39803 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:43:17,021 DEBUG [M:0;e14ef07817f4:39803 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:43:17,021 DEBUG [M:0;e14ef07817f4:39803 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:43:17,021 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131752243 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131752243,5,FailOnTimeoutGroup]
2024-11-20T19:43:17,021 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131752243 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131752243,5,FailOnTimeoutGroup]
2024-11-20T19:43:17,021 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:43:17,022 INFO  [M:0;e14ef07817f4:39803 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:43:17,022 INFO  [M:0;e14ef07817f4:39803 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:43:17,022 DEBUG [M:0;e14ef07817f4:39803 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:43:17,022 INFO  [M:0;e14ef07817f4:39803 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:43:17,023 INFO  [M:0;e14ef07817f4:39803 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:43:17,023 INFO  [M:0;e14ef07817f4:39803 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:43:17,023 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:43:17,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:43:17,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:17,031 DEBUG [M:0;e14ef07817f4:39803 {}] zookeeper.ZKUtil(347): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:43:17,031 WARN  [M:0;e14ef07817f4:39803 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:43:17,032 INFO  [M:0;e14ef07817f4:39803 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/.lastflushedseqids
2024-11-20T19:43:17,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741905_1091 (size=130)
2024-11-20T19:43:17,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741905_1091 (size=130)
2024-11-20T19:43:17,042 INFO  [M:0;e14ef07817f4:39803 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:43:17,043 INFO  [M:0;e14ef07817f4:39803 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:43:17,043 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:43:17,043 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:17,043 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:17,043 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:43:17,043 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:17,043 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-20T19:43:17,061 DEBUG [M:0;e14ef07817f4:39803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91ac769062d1417784472ba99561307e is 82, key is hbase:meta,,1/info:regioninfo/1732131753044/Put/seqid=0
2024-11-20T19:43:17,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741906_1092 (size=5672)
2024-11-20T19:43:17,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741906_1092 (size=5672)
2024-11-20T19:43:17,067 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91ac769062d1417784472ba99561307e
2024-11-20T19:43:17,087 DEBUG [M:0;e14ef07817f4:39803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4015d8413cd4c30b26aee5aa3330501 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732131753768/Put/seqid=0
2024-11-20T19:43:17,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741907_1093 (size=6256)
2024-11-20T19:43:17,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741907_1093 (size=6256)
2024-11-20T19:43:17,092 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4015d8413cd4c30b26aee5aa3330501
2024-11-20T19:43:17,098 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4015d8413cd4c30b26aee5aa3330501
2024-11-20T19:43:17,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:17,110 INFO  [RS:0;e14ef07817f4:37815 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:43:17,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37815-0x10159c502510001, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:17,110 INFO  [RS:0;e14ef07817f4:37815 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,37815,1732131751777; zookeeper connection closed.
2024-11-20T19:43:17,110 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a231aa3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a231aa3
2024-11-20T19:43:17,110 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-20T19:43:17,113 DEBUG [M:0;e14ef07817f4:39803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9161c1057760469b855a21001e523d53 is 69, key is e14ef07817f4,37815,1732131751777/rs:state/1732131752350/Put/seqid=0
2024-11-20T19:43:17,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741908_1094 (size=5224)
2024-11-20T19:43:17,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741908_1094 (size=5224)
2024-11-20T19:43:17,118 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9161c1057760469b855a21001e523d53
2024-11-20T19:43:17,136 DEBUG [M:0;e14ef07817f4:39803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a333980edc254408823f7cc91a4fde2c is 52, key is load_balancer_on/state:d/1732131753223/Put/seqid=0
2024-11-20T19:43:17,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741909_1095 (size=5056)
2024-11-20T19:43:17,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741909_1095 (size=5056)
2024-11-20T19:43:17,141 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a333980edc254408823f7cc91a4fde2c
2024-11-20T19:43:17,146 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/91ac769062d1417784472ba99561307e as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/91ac769062d1417784472ba99561307e
2024-11-20T19:43:17,152 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/91ac769062d1417784472ba99561307e, entries=8, sequenceid=60, filesize=5.5 K
2024-11-20T19:43:17,153 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4015d8413cd4c30b26aee5aa3330501 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f4015d8413cd4c30b26aee5aa3330501
2024-11-20T19:43:17,159 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4015d8413cd4c30b26aee5aa3330501
2024-11-20T19:43:17,159 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f4015d8413cd4c30b26aee5aa3330501, entries=6, sequenceid=60, filesize=6.1 K
2024-11-20T19:43:17,160 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9161c1057760469b855a21001e523d53 as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9161c1057760469b855a21001e523d53
2024-11-20T19:43:17,166 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9161c1057760469b855a21001e523d53, entries=2, sequenceid=60, filesize=5.1 K
2024-11-20T19:43:17,167 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a333980edc254408823f7cc91a4fde2c as hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a333980edc254408823f7cc91a4fde2c
2024-11-20T19:43:17,173 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a333980edc254408823f7cc91a4fde2c, entries=1, sequenceid=60, filesize=4.9 K
2024-11-20T19:43:17,175 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=60, compaction requested=false
2024-11-20T19:43:17,176 INFO  [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:17,176 DEBUG [M:0;e14ef07817f4:39803 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131797043Disabling compacts and flushes for region at 1732131797043Disabling writes for close at 1732131797043Obtaining lock to block concurrent updates at 1732131797043Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131797043Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1732131797043Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131797044 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131797044Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131797061 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131797061Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131797072 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131797086 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131797086Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131797098 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131797113 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131797113Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131797123 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131797136 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131797136Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d86dfc7: reopening flushed file at 1732131797146 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eaaadda: reopening flushed file at 1732131797152 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52940807: reopening flushed file at 1732131797159 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7df8d670: reopening flushed file at 1732131797166 (+7 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 131ms, sequenceid=60, compaction requested=false at 1732131797175 (+9 ms)Writing region close event to WAL at 1732131797176 (+1 ms)Closed at 1732131797176
2024-11-20T19:43:17,177 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:17,177 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:17,177 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:17,177 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:17,177 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:17,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36869 is added to blk_1073741888_1071 (size=1045)
2024-11-20T19:43:17,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741888_1071 (size=1045)
2024-11-20T19:43:17,367 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:43:17,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,388 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,391 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:17,398 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T19:43:17,398 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:43:17,398 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:43:17,398 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T19:43:17,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:17,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:18,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:43:18,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44123 is added to blk_1073741836_1012 (size=76)
2024-11-20T19:43:18,190 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@cd4cd79 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-854921811-172.17.0.3-1732131749074:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:32909,null,null])
java.net.ConnectException: Call From e14ef07817f4/172.17.0.3 to localhost:46447 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T19:43:18,280 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/WALs/e14ef07817f4,39803,1732131751605/e14ef07817f4%2C39803%2C1732131751605.1732131751999 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/oldWALs/e14ef07817f4%2C39803%2C1732131751605.1732131751999
2024-11-20T19:43:18,288 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/MasterData/oldWALs/e14ef07817f4%2C39803%2C1732131751605.1732131751999 to hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/oldWALs/e14ef07817f4%2C39803%2C1732131751605.1732131751999$masterlocalwal$
2024-11-20T19:43:18,289 INFO  [M:0;e14ef07817f4:39803 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:43:18,289 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:43:18,289 INFO  [M:0;e14ef07817f4:39803 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39803
2024-11-20T19:43:18,289 INFO  [M:0;e14ef07817f4:39803 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:43:18,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:18,458 INFO  [M:0;e14ef07817f4:39803 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:43:18,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39803-0x10159c502510000, quorum=127.0.0.1:62155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:18,463 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f32fff4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:18,464 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@434810ac{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:18,464 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:18,465 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c8ca2dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:18,465 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2586080f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:18,467 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:18,467 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid 501d35c9-f0fe-4e3c-ab5e-a76ce562e2f9) service to localhost/127.0.0.1:37581
2024-11-20T19:43:18,466 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@74bebcc1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:32909,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:46447 , LocalHost:localPort e14ef07817f4/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T19:43:18,467 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@74bebcc1 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-854921811-172.17.0.3-1732131749074:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:36869,null,null], DatanodeInfoWithStorage[127.0.0.1:32909,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-854921811-172.17.0.3-1732131749074
2024-11-20T19:43:18,467 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@74bebcc1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:32909,null,null])
java.io.IOException: No block pool offer service for bpid=BP-854921811-172.17.0.3-1732131749074
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:18,467 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data3/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:18,467 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@74bebcc1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:36869,null,null])
java.io.IOException: No block pool offer service for bpid=BP-854921811-172.17.0.3-1732131749074
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:18,468 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@74bebcc1 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:32909,null,null], DatanodeInfoWithStorage[127.0.0.1:36869,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-854921811-172.17.0.3-1732131749074:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:32909,null,null], DatanodeInfoWithStorage[127.0.0.1:36869,null,null]]
2024-11-20T19:43:18,468 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data4/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:18,468 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:18,468 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:18,468 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:18,471 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b65aae1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:18,471 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e08dd81{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:18,471 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:18,471 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37673872{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:18,471 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@136e75a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:18,473 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:18,473 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:18,473 WARN  [BP-854921811-172.17.0.3-1732131749074 heartbeating to localhost/127.0.0.1:37581 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-854921811-172.17.0.3-1732131749074 (Datanode Uuid eb7ba641-5546-4742-a765-e19409ef6022) service to localhost/127.0.0.1:37581
2024-11-20T19:43:18,473 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:18,473 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data9/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:18,474 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/cluster_1946c09d-e3cd-d376-ec5a-50f0dc9282e6/data/data10/current/BP-854921811-172.17.0.3-1732131749074 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:18,474 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:18,479 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2606b08f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:43:18,480 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c053989{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:18,480 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:18,480 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3150e6db{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:18,480 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa07d80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:18,487 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:43:18,516 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:43:18,524 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 79)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37581 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37581
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37581
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37581
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:41931
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:37581
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37581
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6d3cbefdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37581 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37581
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37581
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37581 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6d3cbefdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41931
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37581 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=448 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=53 (was 75), ProcessCount=11 (was 11), AvailableMemoryMB=7415 (was 7855)
2024-11-20T19:43:18,530 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=53, ProcessCount=11, AvailableMemoryMB=7415
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.log.dir so I do NOT create it in target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/9c8bc02c-b385-e94d-a8c6-c6b843c8f867/hadoop.tmp.dir so I do NOT create it in target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5, deleteOnExit=true
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/test.cache.data in system properties and HBase conf
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:43:18,531 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:43:18,532 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:43:18,532 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:43:18,533 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:43:18,533 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:43:18,544 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:43:18,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:18,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:18,924 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:18,928 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:18,930 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:18,930 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:18,930 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:18,930 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:18,931 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@37ba1ac4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:18,931 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42369482{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:19,021 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@482b646b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-43993-hadoop-hdfs-3_4_1-tests_jar-_-any-11681794167583604370/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:43:19,022 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c7a3196{HTTP/1.1, (http/1.1)}{localhost:43993}
2024-11-20T19:43:19,022 INFO  [Time-limited test {}] server.Server(415): Started @158096ms
2024-11-20T19:43:19,032 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:43:19,292 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:19,297 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:19,298 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:19,298 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:19,298 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:19,299 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34c1099f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:19,299 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d6935b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:19,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a16b3f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-46821-hadoop-hdfs-3_4_1-tests_jar-_-any-1177436717953281740/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:19,396 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27ffc774{HTTP/1.1, (http/1.1)}{localhost:46821}
2024-11-20T19:43:19,396 INFO  [Time-limited test {}] server.Server(415): Started @158470ms
2024-11-20T19:43:19,397 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:19,426 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:19,429 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:19,432 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:19,432 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:19,433 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:19,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1118a265{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:19,434 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b91ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:19,525 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27a17641{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-37551-hadoop-hdfs-3_4_1-tests_jar-_-any-12037042974912221365/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:19,526 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@62cd82cb{HTTP/1.1, (http/1.1)}{localhost:37551}
2024-11-20T19:43:19,526 INFO  [Time-limited test {}] server.Server(415): Started @158600ms
2024-11-20T19:43:19,527 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:19,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:19,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:20,740 WARN  [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data1/current/BP-1075613540-172.17.0.3-1732131798548/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:20,740 WARN  [Thread-1203 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data2/current/BP-1075613540-172.17.0.3-1732131798548/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:20,758 WARN  [Thread-1166 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:20,761 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb36143c789b0a448 with lease ID 0x5d38f443db824967: Processing first storage report for DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26 from datanode DatanodeRegistration(127.0.0.1:34583, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44033, infoSecurePort=0, ipcPort=42565, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548)
2024-11-20T19:43:20,761 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb36143c789b0a448 with lease ID 0x5d38f443db824967: from storage DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26 node DatanodeRegistration(127.0.0.1:34583, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44033, infoSecurePort=0, ipcPort=42565, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:20,761 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb36143c789b0a448 with lease ID 0x5d38f443db824967: Processing first storage report for DS-dd4c0394-7cc7-4ca2-b695-989412798a1c from datanode DatanodeRegistration(127.0.0.1:34583, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44033, infoSecurePort=0, ipcPort=42565, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548)
2024-11-20T19:43:20,761 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb36143c789b0a448 with lease ID 0x5d38f443db824967: from storage DS-dd4c0394-7cc7-4ca2-b695-989412798a1c node DatanodeRegistration(127.0.0.1:34583, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44033, infoSecurePort=0, ipcPort=42565, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:20,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:20,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:20,895 WARN  [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data3/current/BP-1075613540-172.17.0.3-1732131798548/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:20,895 WARN  [Thread-1214 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data4/current/BP-1075613540-172.17.0.3-1732131798548/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:20,915 WARN  [Thread-1189 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:20,917 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd224b35956a96ca4 with lease ID 0x5d38f443db824968: Processing first storage report for DS-31fe7848-3f97-44c9-b94f-c88007955fcc from datanode DatanodeRegistration(127.0.0.1:38303, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=39385, infoSecurePort=0, ipcPort=45533, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548)
2024-11-20T19:43:20,917 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd224b35956a96ca4 with lease ID 0x5d38f443db824968: from storage DS-31fe7848-3f97-44c9-b94f-c88007955fcc node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=39385, infoSecurePort=0, ipcPort=45533, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:20,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd224b35956a96ca4 with lease ID 0x5d38f443db824968: Processing first storage report for DS-6b7a874f-8758-42f6-a21e-a641104039ea from datanode DatanodeRegistration(127.0.0.1:38303, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=39385, infoSecurePort=0, ipcPort=45533, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548)
2024-11-20T19:43:20,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd224b35956a96ca4 with lease ID 0x5d38f443db824968: from storage DS-6b7a874f-8758-42f6-a21e-a641104039ea node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=39385, infoSecurePort=0, ipcPort=45533, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:20,965 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120
2024-11-20T19:43:20,969 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/zookeeper_0, clientPort=64289, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:43:20,970 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64289
2024-11-20T19:43:20,971 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:20,973 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:20,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:43:20,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:43:20,984 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6 with version=8
2024-11-20T19:43:20,984 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:43:20,986 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:43:20,987 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43153
2024-11-20T19:43:20,988 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43153 connecting to ZooKeeper ensemble=127.0.0.1:64289
2024-11-20T19:43:21,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431530x0, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:43:21,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43153-0x10159c5c3330000 connected
2024-11-20T19:43:21,137 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:21,141 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:21,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:21,145 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6, hbase.cluster.distributed=false
2024-11-20T19:43:21,147 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:43:21,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43153
2024-11-20T19:43:21,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43153
2024-11-20T19:43:21,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43153
2024-11-20T19:43:21,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43153
2024-11-20T19:43:21,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43153
2024-11-20T19:43:21,163 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:43:21,164 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:43:21,165 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45591
2024-11-20T19:43:21,166 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45591 connecting to ZooKeeper ensemble=127.0.0.1:64289
2024-11-20T19:43:21,166 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:21,168 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:21,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455910x0, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:43:21,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45591-0x10159c5c3330001 connected
2024-11-20T19:43:21,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:21,179 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:43:21,179 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:43:21,180 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:43:21,181 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:43:21,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45591
2024-11-20T19:43:21,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45591
2024-11-20T19:43:21,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45591
2024-11-20T19:43:21,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45591
2024-11-20T19:43:21,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45591
2024-11-20T19:43:21,195 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:43153
2024-11-20T19:43:21,195 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:21,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:21,207 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:43:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,221 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:43:21,222 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,43153,1732131800986 from backup master directory
2024-11-20T19:43:21,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:21,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:21,231 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:43:21,231 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/hbase.id] with ID: e3e00913-989d-4c33-8992-ffe5ec5bfb95
2024-11-20T19:43:21,240 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/.tmp/hbase.id
2024-11-20T19:43:21,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:43:21,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:43:21,246 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/.tmp/hbase.id]:[hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/hbase.id]
2024-11-20T19:43:21,258 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:21,258 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:43:21,259 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T19:43:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,282 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:43:21,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:43:21,283 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:43:21,284 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:43:21,285 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:21,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:43:21,292 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:43:21,293 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store
2024-11-20T19:43:21,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:43:21,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:43:21,299 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:21,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:43:21,300 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:21,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:21,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:43:21,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:21,300 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:21,300 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131801300Disabling compacts and flushes for region at 1732131801300Disabling writes for close at 1732131801300Writing region close event to WAL at 1732131801300Closed at 1732131801300
2024-11-20T19:43:21,301 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/.initializing
2024-11-20T19:43:21,301 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,303 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C43153%2C1732131800986, suffix=, logDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986, archiveDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/oldWALs, maxLogs=10
2024-11-20T19:43:21,303 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C43153%2C1732131800986.1732131801303
2024-11-20T19:43:21,308 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303
2024-11-20T19:43:21,312 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44033:44033),(127.0.0.1/127.0.0.1:39385:39385)]
2024-11-20T19:43:21,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:43:21,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:21,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,313 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,315 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:21,316 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:43:21,316 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,316 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:21,317 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:21,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:43:21,318 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:21,318 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:21,319 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:43:21,319 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,320 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:21,320 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:21,321 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:43:21,321 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,321 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:21,322 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,322 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,322 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,323 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,324 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,324 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:43:21,325 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:21,329 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:43:21,329 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=697966, jitterRate=-0.11249104142189026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:43:21,329 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131801314Initializing all the Stores at 1732131801314Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131801315 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131801315Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131801315Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131801315Cleaning up temporary data from old regions at 1732131801324 (+9 ms)Region opened successfully at 1732131801329 (+5 ms)
2024-11-20T19:43:21,330 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:43:21,333 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e94dbbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:43:21,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:43:21,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:43:21,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:43:21,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:43:21,334 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:43:21,335 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:43:21,335 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:43:21,336 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:43:21,337 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:43:21,346 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:43:21,346 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:43:21,347 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:43:21,357 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:43:21,357 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:43:21,358 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:43:21,367 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:43:21,369 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:43:21,378 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:43:21,380 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:43:21,388 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:43:21,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:21,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:21,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,399 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,43153,1732131800986, sessionid=0x10159c5c3330000, setting cluster-up flag (Was=false)
2024-11-20T19:43:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,451 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:43:21,453 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:21,504 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:43:21,506 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,43153,1732131800986
2024-11-20T19:43:21,507 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:43:21,508 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:21,509 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:43:21,509 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:43:21,509 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,43153,1732131800986
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:43:21,511 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,512 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131831512
2024-11-20T19:43:21,512 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,513 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:43:21,513 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:43:21,513 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:43:21,514 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131801514,5,FailOnTimeoutGroup]
2024-11-20T19:43:21,514 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131801514,5,FailOnTimeoutGroup]
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,514 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,514 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,514 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:43:21,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:43:21,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:43:21,584 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(746): ClusterId : e3e00913-989d-4c33-8992-ffe5ec5bfb95
2024-11-20T19:43:21,584 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:43:21,597 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:43:21,597 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:43:21,611 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:43:21,611 DEBUG [RS:0;e14ef07817f4:45591 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3429f3ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:43:21,633 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:45591
2024-11-20T19:43:21,633 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:43:21,633 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:43:21,633 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:43:21,634 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,43153,1732131800986 with port=45591, startcode=1732131801163
2024-11-20T19:43:21,634 DEBUG [RS:0;e14ef07817f4:45591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:43:21,636 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39145, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:43:21,636 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43153 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,636 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43153 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,638 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6
2024-11-20T19:43:21,638 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36317
2024-11-20T19:43:21,638 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:43:21,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:43:21,649 DEBUG [RS:0;e14ef07817f4:45591 {}] zookeeper.ZKUtil(111): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,649 WARN  [RS:0;e14ef07817f4:45591 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:43:21,649 INFO  [RS:0;e14ef07817f4:45591 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:21,649 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,650 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,45591,1732131801163]
2024-11-20T19:43:21,653 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:43:21,654 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:43:21,654 INFO  [RS:0;e14ef07817f4:45591 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:43:21,654 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,655 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:43:21,655 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:43:21,655 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:43:21,656 DEBUG [RS:0;e14ef07817f4:45591 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,658 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,45591,1732131801163-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:43:21,671 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:43:21,671 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,45591,1732131801163-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,671 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,671 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.Replication(171): e14ef07817f4,45591,1732131801163 started
2024-11-20T19:43:21,684 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:21,684 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,45591,1732131801163, RpcServer on e14ef07817f4/172.17.0.3:45591, sessionid=0x10159c5c3330001
2024-11-20T19:43:21,685 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:43:21,685 DEBUG [RS:0;e14ef07817f4:45591 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,685 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,45591,1732131801163'
2024-11-20T19:43:21,685 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:43:21,685 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,45591,1732131801163
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,45591,1732131801163'
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:43:21,686 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:43:21,687 DEBUG [RS:0;e14ef07817f4:45591 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:43:21,687 INFO  [RS:0;e14ef07817f4:45591 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:43:21,687 INFO  [RS:0;e14ef07817f4:45591 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:43:21,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:21,790 INFO  [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C45591%2C1732131801163, suffix=, logDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163, archiveDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs, maxLogs=32
2024-11-20T19:43:21,792 INFO  [RS:0;e14ef07817f4:45591 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:21,802 INFO  [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:21,802 DEBUG [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39385:39385),(127.0.0.1/127.0.0.1:44033:44033)]
2024-11-20T19:43:21,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:21,923 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:43:21,923 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6
2024-11-20T19:43:21,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741833_1009 (size=32)
2024-11-20T19:43:21,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741833_1009 (size=32)
2024-11-20T19:43:21,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:21,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:43:21,937 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:43:21,937 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:21,938 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:43:21,939 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:43:21,939 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,939 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:21,940 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:43:21,941 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:43:21,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:21,941 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:43:21,942 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:43:21,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:21,943 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:21,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:43:21,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740
2024-11-20T19:43:21,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740
2024-11-20T19:43:21,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:43:21,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:43:21,945 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:43:21,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:43:21,948 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:43:21,949 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=765922, jitterRate=-0.026081055402755737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:43:21,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131801935Initializing all the Stores at 1732131801935Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131801935Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131801936 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131801936Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131801936Cleaning up temporary data from old regions at 1732131801945 (+9 ms)Region opened successfully at 1732131801949 (+4 ms)
2024-11-20T19:43:21,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:43:21,949 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:43:21,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:43:21,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:43:21,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:43:21,950 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:21,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131801949Disabling compacts and flushes for region at 1732131801949Disabling writes for close at 1732131801949Writing region close event to WAL at 1732131801950 (+1 ms)Closed at 1732131801950
2024-11-20T19:43:21,951 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:21,951 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:43:21,951 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:43:21,953 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:43:21,954 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:43:22,104 DEBUG [e14ef07817f4:43153 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:43:22,105 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:22,108 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,45591,1732131801163, state=OPENING
2024-11-20T19:43:22,165 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:43:22,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:22,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:22,178 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:43:22,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:22,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:22,178 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,45591,1732131801163}]
2024-11-20T19:43:22,332 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:43:22,337 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37325, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:43:22,344 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:43:22,345 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:22,347 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C45591%2C1732131801163.meta, suffix=.meta, logDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163, archiveDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs, maxLogs=32
2024-11-20T19:43:22,347 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta
2024-11-20T19:43:22,353 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta
2024-11-20T19:43:22,356 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44033:44033),(127.0.0.1/127.0.0.1:39385:39385)]
2024-11-20T19:43:22,360 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:43:22,360 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:43:22,361 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:43:22,361 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:43:22,361 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:43:22,361 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:22,361 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:43:22,361 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:43:22,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:43:22,363 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:43:22,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:22,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:43:22,365 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:43:22,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,365 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:22,366 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:43:22,366 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:43:22,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:22,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:43:22,368 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:43:22,368 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,368 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:22,368 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:43:22,369 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740
2024-11-20T19:43:22,370 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740
2024-11-20T19:43:22,371 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:43:22,371 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:43:22,372 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:43:22,373 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:43:22,374 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868468, jitterRate=0.10431475937366486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:43:22,374 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:43:22,374 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131802361Writing region info on filesystem at 1732131802361Initializing all the Stores at 1732131802362 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131802362Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131802362Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131802362Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131802363 (+1 ms)Cleaning up temporary data from old regions at 1732131802371 (+8 ms)Running coprocessor post-open hooks at 1732131802374 (+3 ms)Region opened successfully at 1732131802374
2024-11-20T19:43:22,375 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131802332
2024-11-20T19:43:22,377 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:43:22,378 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:43:22,379 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:22,380 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,45591,1732131801163, state=OPEN
2024-11-20T19:43:22,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:43:22,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:43:22,419 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:22,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:22,419 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:22,423 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:43:22,423 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,45591,1732131801163 in 241 msec
2024-11-20T19:43:22,427 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:43:22,427 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 472 msec
2024-11-20T19:43:22,428 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:22,428 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:43:22,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:43:22,429 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,45591,1732131801163, seqNum=-1]
2024-11-20T19:43:22,430 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:43:22,431 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38687, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:43:22,438 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 928 msec
2024-11-20T19:43:22,438 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131802438, completionTime=-1
2024-11-20T19:43:22,438 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:43:22,438 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:43:22,440 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:43:22,440 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131862440
2024-11-20T19:43:22,440 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732131922440
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:43153, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,441 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,442 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,444 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.215sec
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:43:22,446 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:43:22,449 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:43:22,449 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:43:22,450 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,43153,1732131800986-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:22,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b81225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:43:22,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,43153,-1 for getting cluster id
2024-11-20T19:43:22,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:43:22,489 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e3e00913-989d-4c33-8992-ffe5ec5bfb95'
2024-11-20T19:43:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:43:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e3e00913-989d-4c33-8992-ffe5ec5bfb95"

2024-11-20T19:43:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c927b10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:43:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,43153,-1]
2024-11-20T19:43:22,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:43:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:22,494 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:43:22,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5282eca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:43:22,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:43:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,45591,1732131801163, seqNum=-1]
2024-11-20T19:43:22,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:43:22,500 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:43:22,503 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,43153,1732131800986
2024-11-20T19:43:22,504 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:22,507 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:43:22,507 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-20T19:43:22,507 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-20T19:43:22,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T19:43:22,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is e14ef07817f4,43153,1732131800986
2024-11-20T19:43:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@796c0314
2024-11-20T19:43:22,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T19:43:22,511 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T19:43:22,512 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T19:43:22,512 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T19:43:22,512 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:43:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T19:43:22,516 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T19:43:22,516 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,516 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-20T19:43:22,517 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T19:43:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:43:22,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741835_1011 (size=395)
2024-11-20T19:43:22,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741835_1011 (size=395)
2024-11-20T19:43:22,526 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb00cbb03387cfb464cb85ebcd040388, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6
2024-11-20T19:43:22,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34583 is added to blk_1073741836_1012 (size=78)
2024-11-20T19:43:22,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741836_1012 (size=78)
2024-11-20T19:43:22,532 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:22,533 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing cb00cbb03387cfb464cb85ebcd040388, disabling compactions & flushes
2024-11-20T19:43:22,533 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,533 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,533 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388. after waiting 0 ms
2024-11-20T19:43:22,533 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,533 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,533 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb00cbb03387cfb464cb85ebcd040388:
Waiting for close lock at 1732131802533Disabling compacts and flushes for region at 1732131802533Disabling writes for close at 1732131802533Writing region close event to WAL at 1732131802533Closed at 1732131802533
2024-11-20T19:43:22,534 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T19:43:22,534 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732131802534"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131802534"}]},"ts":"1732131802534"}
2024-11-20T19:43:22,537 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T19:43:22,538 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T19:43:22,538 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131802538"}]},"ts":"1732131802538"}
2024-11-20T19:43:22,540 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-20T19:43:22,540 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=cb00cbb03387cfb464cb85ebcd040388, ASSIGN}]
2024-11-20T19:43:22,542 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=cb00cbb03387cfb464cb85ebcd040388, ASSIGN
2024-11-20T19:43:22,543 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=cb00cbb03387cfb464cb85ebcd040388, ASSIGN; state=OFFLINE, location=e14ef07817f4,45591,1732131801163; forceNewPlan=false, retain=false
2024-11-20T19:43:22,695 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb00cbb03387cfb464cb85ebcd040388, regionState=OPENING, regionLocation=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:22,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=cb00cbb03387cfb464cb85ebcd040388, ASSIGN because future has completed
2024-11-20T19:43:22,702 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb00cbb03387cfb464cb85ebcd040388, server=e14ef07817f4,45591,1732131801163}]
2024-11-20T19:43:22,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:22,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:22,862 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,862 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cb00cbb03387cfb464cb85ebcd040388, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:43:22,862 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,863 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:22,863 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,863 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,866 INFO  [StoreOpener-cb00cbb03387cfb464cb85ebcd040388-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cb00cbb03387cfb464cb85ebcd040388 
2024-11-20T19:43:22,869 INFO  [StoreOpener-cb00cbb03387cfb464cb85ebcd040388-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb00cbb03387cfb464cb85ebcd040388 columnFamilyName info
2024-11-20T19:43:22,869 DEBUG [StoreOpener-cb00cbb03387cfb464cb85ebcd040388-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:22,870 INFO  [StoreOpener-cb00cbb03387cfb464cb85ebcd040388-1 {}] regionserver.HStore(327): Store=cb00cbb03387cfb464cb85ebcd040388/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:22,870 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,872 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,873 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,874 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,874 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,877 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,879 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:43:22,879 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cb00cbb03387cfb464cb85ebcd040388; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841071, jitterRate=0.06947717070579529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:43:22,880 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:22,880 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cb00cbb03387cfb464cb85ebcd040388:
Running coprocessor pre-open hook at 1732131802863Writing region info on filesystem at 1732131802863Initializing all the Stores at 1732131802865 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131802865Cleaning up temporary data from old regions at 1732131802874 (+9 ms)Running coprocessor post-open hooks at 1732131802880 (+6 ms)Region opened successfully at 1732131802880
2024-11-20T19:43:22,881 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388., pid=6, masterSystemTime=1732131802855
2024-11-20T19:43:22,884 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,884 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:22,885 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb00cbb03387cfb464cb85ebcd040388, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,45591,1732131801163
2024-11-20T19:43:22,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb00cbb03387cfb464cb85ebcd040388, server=e14ef07817f4,45591,1732131801163 because future has completed
2024-11-20T19:43:22,891 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T19:43:22,892 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cb00cbb03387cfb464cb85ebcd040388, server=e14ef07817f4,45591,1732131801163 in 186 msec
2024-11-20T19:43:22,895 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T19:43:22,895 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=cb00cbb03387cfb464cb85ebcd040388, ASSIGN in 352 msec
2024-11-20T19:43:22,896 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T19:43:22,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131802896"}]},"ts":"1732131802896"}
2024-11-20T19:43:22,899 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-20T19:43:22,900 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:43:22,900 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T19:43:22,902 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 388 msec
2024-11-20T19:43:22,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,920 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,920 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:22,927 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:23,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:23,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:24,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:24,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:25,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:25,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:26,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:26,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:27,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:27,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:27,866 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:43:27,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:27,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:43:27,896 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T19:43:27,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T19:43:27,896 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-20T19:43:27,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:43:27,897 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T19:43:27,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T19:43:27,897 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-20T19:43:27,897 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:43:27,898 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-20T19:43:28,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:28,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:29,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:29,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:30,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:30,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:31,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:31,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43153 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:43:32,617 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-20T19:43:32,618 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-20T19:43:32,626 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T19:43:32,626 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388., hostname=e14ef07817f4,45591,1732131801163, seqNum=2]
2024-11-20T19:43:32,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:32,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:33,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:33,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:34,633 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:34,635 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,635 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,635 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,636 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK], DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]) is bad.
2024-11-20T19:43:34,636 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK], DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]) is bad.
2024-11-20T19:43:34,636 WARN  [PacketResponder: BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38303] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,636 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK], DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38303,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]) is bad.
2024-11-20T19:43:34,636 WARN  [PacketResponder: BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:38303] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,637 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:41722 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41722 dst: /127.0.0.1:34583
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,637 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_499351357_22 at /127.0.0.1:56546 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56546 dst: /127.0.0.1:34583
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,637 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:45406 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:38303:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45406 dst: /127.0.0.1:38303
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:56566 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56566 dst: /127.0.0.1:34583
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:44858 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:38303:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44858 dst: /127.0.0.1:38303
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_499351357_22 at /127.0.0.1:45388 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:38303:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45388 dst: /127.0.0.1:38303
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27a17641{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:34,671 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@62cd82cb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:34,671 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:34,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b91ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:34,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1118a265{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:34,673 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:34,673 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:34,673 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid 78cf0a18-0a10-4f53-bc0c-4f4b3f88013b) service to localhost/127.0.0.1:36317
2024-11-20T19:43:34,673 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:34,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data3/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:34,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data4/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:34,675 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:34,685 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:34,688 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:34,689 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:34,689 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:34,689 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:43:34,689 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fa662a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:34,690 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e3b369c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:34,785 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7c83d523{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-44625-hadoop-hdfs-3_4_1-tests_jar-_-any-13100397081600821358/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:34,786 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@341e0a7a{HTTP/1.1, (http/1.1)}{localhost:44625}
2024-11-20T19:43:34,786 INFO  [Time-limited test {}] server.Server(415): Started @173860ms
2024-11-20T19:43:34,787 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:34,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:34,805 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,805 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,805 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:34,805 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_499351357_22 at /127.0.0.1:51124 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51124 dst: /127.0.0.1:34583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,805 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:51116 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51116 dst: /127.0.0.1:34583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,806 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:51108 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51108 dst: /127.0.0.1:34583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:34,811 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a16b3f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:34,812 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27ffc774{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:34,812 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:34,812 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d6935b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:34,812 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34c1099f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:34,813 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:34,813 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid a89e6e79-7275-47b4-8d5f-af5885cf361d) service to localhost/127.0.0.1:36317
2024-11-20T19:43:34,814 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data1/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:34,814 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data2/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:34,814 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:34,814 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:34,814 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:34,823 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:34,826 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:34,827 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:34,827 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:34,827 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:43:34,828 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20466b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:34,828 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29d5ccf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:34,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:34,948 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42bf2aaa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-35993-hadoop-hdfs-3_4_1-tests_jar-_-any-6419519945739425050/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:34,948 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9fe4e76{HTTP/1.1, (http/1.1)}{localhost:35993}
2024-11-20T19:43:34,948 INFO  [Time-limited test {}] server.Server(415): Started @174022ms
2024-11-20T19:43:34,949 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:35,433 WARN  [Thread-1337 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:35,436 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26aca430abf5306 with lease ID 0x5d38f443db824969: from storage DS-31fe7848-3f97-44c9-b94f-c88007955fcc node DatanodeRegistration(127.0.0.1:36911, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=35685, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:35,436 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26aca430abf5306 with lease ID 0x5d38f443db824969: from storage DS-6b7a874f-8758-42f6-a21e-a641104039ea node DatanodeRegistration(127.0.0.1:36911, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=35685, infoSecurePort=0, ipcPort=39887, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:43:35,546 WARN  [Thread-1357 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:35,548 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ae85a5d3d802b52 with lease ID 0x5d38f443db82496a: from storage DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26 node DatanodeRegistration(127.0.0.1:34307, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44529, infoSecurePort=0, ipcPort=37339, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:35,549 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ae85a5d3d802b52 with lease ID 0x5d38f443db82496a: from storage DS-dd4c0394-7cc7-4ca2-b695-989412798a1c node DatanodeRegistration(127.0.0.1:34307, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=44529, infoSecurePort=0, ipcPort=37339, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:35,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:35,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:35,966 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-20T19:43:35,974 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-20T19:43:35,976 ERROR [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:35,977 WARN  [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:35,977 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C45591%2C1732131801163:(num 1732131801791) roll requested
2024-11-20T19:43:35,977 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:35,984 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 newFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:35,984 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:35,984 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:35,984 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:35,985 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:35,985 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:35,985 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:35,987 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:35,987 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:35,987 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:35,987 WARN  [IPC Server handler 3 on default port 36317 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741832_1014
2024-11-20T19:43:35,988 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 after 1ms
2024-11-20T19:43:35,988 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35685:35685),(127.0.0.1/127.0.0.1:44529:44529)]
2024-11-20T19:43:35,988 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 is not closed yet, will try archiving it next time
2024-11-20T19:43:36,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:36,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:37,437 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741832_1014: GenerationStamp not matched, existing replica is blk_1073741832_1008
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T19:43:37,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:37,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:37,994 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-20T19:43:38,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:38,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:39,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:39,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:39,990 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 after 4003ms
2024-11-20T19:43:40,002 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:34307,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:40,003 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36911,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK], DatanodeInfoWithStorage[127.0.0.1:34307,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34307,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]) is bad.
2024-11-20T19:43:40,003 WARN  [PacketResponder: BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:34307] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:40,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:33606 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36911:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33606 dst: /127.0.0.1:36911
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:40,004 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:58430 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:34307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58430 dst: /127.0.0.1:34307
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:40,066 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42bf2aaa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:40,066 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9fe4e76{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:40,067 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:40,067 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29d5ccf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:40,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20466b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:40,070 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:40,070 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:40,071 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid a89e6e79-7275-47b4-8d5f-af5885cf361d) service to localhost/127.0.0.1:36317
2024-11-20T19:43:40,071 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:40,072 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data1/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:40,073 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data2/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:40,073 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:40,083 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:40,086 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:40,087 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:40,087 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:40,087 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:43:40,087 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ae70be3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:40,088 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58c81822{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:40,180 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c3c718b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-37455-hadoop-hdfs-3_4_1-tests_jar-_-any-6698050681465066438/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:40,180 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7df90452{HTTP/1.1, (http/1.1)}{localhost:37455}
2024-11-20T19:43:40,180 INFO  [Time-limited test {}] server.Server(415): Started @179254ms
2024-11-20T19:43:40,181 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:40,195 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:40,196 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1473359295_22 at /127.0.0.1:33622 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36911:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33622 dst: /127.0.0.1:36911
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:40,197 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7c83d523{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:40,198 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@341e0a7a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:40,198 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:40,198 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e3b369c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:40,198 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fa662a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:40,199 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:40,199 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:40,199 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid 78cf0a18-0a10-4f53-bc0c-4f4b3f88013b) service to localhost/127.0.0.1:36317
2024-11-20T19:43:40,199 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:40,199 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data3/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:40,199 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data4/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:40,200 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:40,210 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:40,213 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:40,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:40,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:40,214 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:40,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78f99ac1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:40,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7202b0aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:40,307 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@649d2ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/java.io.tmpdir/jetty-localhost-39795-hadoop-hdfs-3_4_1-tests_jar-_-any-15692240875036306931/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:40,307 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1b0a2346{HTTP/1.1, (http/1.1)}{localhost:39795}
2024-11-20T19:43:40,307 INFO  [Time-limited test {}] server.Server(415): Started @179382ms
2024-11-20T19:43:40,309 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:40,774 WARN  [Thread-1411 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:40,776 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe40cc20ed0f8aad9 with lease ID 0x5d38f443db82496b: from storage DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26 node DatanodeRegistration(127.0.0.1:40215, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=41253, infoSecurePort=0, ipcPort=36763, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:40,776 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe40cc20ed0f8aad9 with lease ID 0x5d38f443db82496b: from storage DS-dd4c0394-7cc7-4ca2-b695-989412798a1c node DatanodeRegistration(127.0.0.1:40215, datanodeUuid=a89e6e79-7275-47b4-8d5f-af5885cf361d, infoPort=41253, infoSecurePort=0, ipcPort=36763, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:40,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:40,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:40,875 WARN  [Thread-1431 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:40,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8378a4b277d12e89 with lease ID 0x5d38f443db82496c: from storage DS-31fe7848-3f97-44c9-b94f-c88007955fcc node DatanodeRegistration(127.0.0.1:37515, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=42673, infoSecurePort=0, ipcPort=34297, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:40,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8378a4b277d12e89 with lease ID 0x5d38f443db82496c: from storage DS-6b7a874f-8758-42f6-a21e-a641104039ea node DatanodeRegistration(127.0.0.1:37515, datanodeUuid=78cf0a18-0a10-4f53-bc0c-4f4b3f88013b, infoPort=42673, infoSecurePort=0, ipcPort=34297, storageInfo=lv=-57;cid=testClusterID;nsid=439817694;c=1732131798548), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:41,327 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-20T19:43:41,331 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-20T19:43:41,334 ERROR [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36911,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:41,335 WARN  [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36911,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:41,335 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C45591%2C1732131801163:(num 1732131815977) roll requested
2024-11-20T19:43:41,335 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:41,345 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 newFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:41,345 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:41,345 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:41,345 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:41,345 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:41,345 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:41,346 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:41,346 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36911,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:41,346 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36911,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:41,346 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:41,346 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41253:41253),(127.0.0.1/127.0.0.1:42673:42673)]
2024-11-20T19:43:41,346 WARN  [IPC Server handler 4 on default port 36317 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-20T19:43:41,346 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 is not closed yet, will try archiving it next time
2024-11-20T19:43:41,347 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 after 1ms
2024-11-20T19:43:41,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:41,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:42,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:42,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:43,348 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:43,361 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335 newFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:43,362 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:43,362 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:43,362 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:43,362 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:43,363 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:43,363 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:43,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741838_1019 (size=1264)
2024-11-20T19:43:43,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741838_1019 (size=1264)
2024-11-20T19:43:43,366 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 is not closed yet, will try archiving it next time
2024-11-20T19:43:43,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41253:41253)]
2024-11-20T19:43:43,371 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 is not closed yet, will try archiving it next time
2024-11-20T19:43:43,371 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:43,371 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:43,372 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791 after 0ms
2024-11-20T19:43:43,372 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:43,379 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732131802880/Put/vlen=218/seqid=0]
2024-11-20T19:43:43,380 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732131812631/Put/vlen=1045/seqid=0]
2024-11-20T19:43:43,380 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131801791
2024-11-20T19:43:43,380 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:43,380 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:43,380 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 after 0ms
2024-11-20T19:43:43,380 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:43,384 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732131815976/Put/vlen=1045/seqid=0]
2024-11-20T19:43:43,384 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732131817997/Put/vlen=1045/seqid=0]
2024-11-20T19:43:43,384 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977
2024-11-20T19:43:43,384 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:43,384 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:43,384 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335 after 0ms
2024-11-20T19:43:43,384 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131821335
2024-11-20T19:43:43,387 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732131821334/Put/vlen=1045/seqid=0]
2024-11-20T19:43:43,388 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:43,388 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:43,388 WARN  [IPC Server handler 2 on default port 36317 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-20T19:43:43,388 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 after 0ms
2024-11-20T19:43:43,780 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T19:43:43,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:43,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:43,881 WARN  [ResponseProcessor for block BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:43,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_499351357_22 at /127.0.0.1:52628 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52628 dst: /127.0.0.1:37515
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37515 remote=/127.0.0.1:52628]. Total timeout mills is 60000, 59481 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:43,881 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_499351357_22 at /127.0.0.1:44390 [Receiving block BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:40215:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44390 dst: /127.0.0.1:40215
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:43:43,882 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 block BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37515,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK], DatanodeInfoWithStorage[127.0.0.1:40215,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37515,DS-31fe7848-3f97-44c9-b94f-c88007955fcc,DISK]) is bad.
2024-11-20T19:43:43,888 WARN  [DataStreamer for file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 block BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:43,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741839_1022 (size=85)
2024-11-20T19:43:43,894 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741839_1022 (size=85)
2024-11-20T19:43:44,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:44,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:45,349 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131815977 after 4003ms
2024-11-20T19:43:45,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:45,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:46,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:46,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:47,390 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 after 4002ms
2024-11-20T19:43:47,390 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:47,398 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:47,399 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-20T19:43:47,399 ERROR [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,400 WARN  [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,400 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C45591%2C1732131801163.meta:.meta(num 1732131802347) roll requested
2024-11-20T19:43:47,400 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.meta.1732131827400.meta
2024-11-20T19:43:47,407 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,407 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,407 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,407 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,407 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,408 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131827400.meta
2024-11-20T19:43:47,408 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,408 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,408 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta
2024-11-20T19:43:47,408 WARN  [IPC Server handler 4 on default port 36317 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-11-20T19:43:47,409 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta after 1ms
2024-11-20T19:43:47,412 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41253:41253)]
2024-11-20T19:43:47,412 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta is not closed yet, will try archiving it next time
2024-11-20T19:43:47,428 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/info/9e78e2157eab4701a19950d2bad821ad is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388./info:regioninfo/1732131802884/Put/seqid=0
2024-11-20T19:43:47,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741841_1025 (size=7125)
2024-11-20T19:43:47,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741841_1025 (size=7125)
2024-11-20T19:43:47,433 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/info/9e78e2157eab4701a19950d2bad821ad
2024-11-20T19:43:47,451 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/ns/800f70fa779449038f0909dd0a90a790 is 43, key is default/ns:d/1732131802432/Put/seqid=0
2024-11-20T19:43:47,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741842_1026 (size=5153)
2024-11-20T19:43:47,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741842_1026 (size=5153)
2024-11-20T19:43:47,455 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/ns/800f70fa779449038f0909dd0a90a790
2024-11-20T19:43:47,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/table/074f4c312d6a44629d621edbceeef54c is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732131802896/Put/seqid=0
2024-11-20T19:43:47,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741843_1027 (size=5438)
2024-11-20T19:43:47,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741843_1027 (size=5438)
2024-11-20T19:43:47,479 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/table/074f4c312d6a44629d621edbceeef54c
2024-11-20T19:43:47,485 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/info/9e78e2157eab4701a19950d2bad821ad as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/info/9e78e2157eab4701a19950d2bad821ad
2024-11-20T19:43:47,490 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/info/9e78e2157eab4701a19950d2bad821ad, entries=10, sequenceid=11, filesize=7.0 K
2024-11-20T19:43:47,491 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/ns/800f70fa779449038f0909dd0a90a790 as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/ns/800f70fa779449038f0909dd0a90a790
2024-11-20T19:43:47,496 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/ns/800f70fa779449038f0909dd0a90a790, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T19:43:47,497 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/.tmp/table/074f4c312d6a44629d621edbceeef54c as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/table/074f4c312d6a44629d621edbceeef54c
2024-11-20T19:43:47,502 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/table/074f4c312d6a44629d621edbceeef54c, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T19:43:47,503 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false
2024-11-20T19:43:47,503 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-20T19:43:47,503 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing cb00cbb03387cfb464cb85ebcd040388 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-20T19:43:47,503 ERROR [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,504 WARN  [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6-prefix:e14ef07817f4,45591,1732131801163 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,504 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C45591%2C1732131801163:(num 1732131823348) roll requested
2024-11-20T19:43:47,504 INFO  [regionserver/e14ef07817f4:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45591%2C1732131801163.1732131827504
2024-11-20T19:43:47,509 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 newFile=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131827504
2024-11-20T19:43:47,509 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,509 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,509 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,509 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,509 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,509 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131827504
2024-11-20T19:43:47,510 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,510 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1075613540-172.17.0.3-1732131798548:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor197.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:47,510 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:47,511 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41253:41253)]
2024-11-20T19:43:47,511 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 after 1ms
2024-11-20T19:43:47,511 DEBUG [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 is not closed yet, will try archiving it next time
2024-11-20T19:43:47,511 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.1732131823348 to hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs/e14ef07817f4%2C45591%2C1732131801163.1732131823348
2024-11-20T19:43:47,524 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/.tmp/info/b68ee5daaae946d5bd9ce38e49a10773 is 1080, key is row1002/info:/1732131812631/Put/seqid=0
2024-11-20T19:43:47,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741845_1029 (size=9270)
2024-11-20T19:43:47,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741845_1029 (size=9270)
2024-11-20T19:43:47,529 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/.tmp/info/b68ee5daaae946d5bd9ce38e49a10773
2024-11-20T19:43:47,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/.tmp/info/b68ee5daaae946d5bd9ce38e49a10773 as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/info/b68ee5daaae946d5bd9ce38e49a10773
2024-11-20T19:43:47,539 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/info/b68ee5daaae946d5bd9ce38e49a10773, entries=4, sequenceid=8, filesize=9.1 K
2024-11-20T19:43:47,540 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for cb00cbb03387cfb464cb85ebcd040388 in 37ms, sequenceid=8, compaction requested=false
2024-11-20T19:43:47,540 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cb00cbb03387cfb464cb85ebcd040388:

2024-11-20T19:43:47,546 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:43:47,546 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:43:47,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:47,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:47,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:47,546 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:43:47,547 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:43:47,547 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1498404850, stopped=false
2024-11-20T19:43:47,547 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,43153,1732131800986
2024-11-20T19:43:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:47,608 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:43:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:47,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:47,609 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:43:47,609 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:47,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:47,609 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:47,609 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:47,610 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,45591,1732131801163' *****
2024-11-20T19:43:47,610 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:43:47,610 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:43:47,610 INFO  [RS:0;e14ef07817f4:45591 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:43:47,610 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:43:47,610 INFO  [RS:0;e14ef07817f4:45591 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:43:47,610 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(3091): Received CLOSE for cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,45591,1732131801163
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:45591.
2024-11-20T19:43:47,611 DEBUG [RS:0;e14ef07817f4:45591 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:43:47,611 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb00cbb03387cfb464cb85ebcd040388, disabling compactions & flushes
2024-11-20T19:43:47,611 DEBUG [RS:0;e14ef07817f4:45591 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:47,611 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:47,611 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:43:47,611 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388. after waiting 0 ms
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:43:47,611 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:43:47,611 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:43:47,612 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T19:43:47,612 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, cb00cbb03387cfb464cb85ebcd040388=TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.}
2024-11-20T19:43:47,612 DEBUG [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cb00cbb03387cfb464cb85ebcd040388
2024-11-20T19:43:47,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:43:47,612 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:43:47,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:43:47,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:43:47,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:43:47,617 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/default/TestLogRolling-testLogRollOnPipelineRestart/cb00cbb03387cfb464cb85ebcd040388/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-20T19:43:47,617 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:47,618 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T19:43:47,618 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb00cbb03387cfb464cb85ebcd040388:
Waiting for close lock at 1732131827611Running coprocessor pre-close hooks at 1732131827611Disabling compacts and flushes for region at 1732131827611Disabling writes for close at 1732131827611Writing region close event to WAL at 1732131827612 (+1 ms)Running coprocessor post-close hooks at 1732131827617 (+5 ms)Closed at 1732131827617
2024-11-20T19:43:47,618 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732131802512.cb00cbb03387cfb464cb85ebcd040388.
2024-11-20T19:43:47,618 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:43:47,618 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:47,618 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131827612Running coprocessor pre-close hooks at 1732131827612Disabling compacts and flushes for region at 1732131827612Disabling writes for close at 1732131827612Writing region close event to WAL at 1732131827613 (+1 ms)Running coprocessor post-close hooks at 1732131827618 (+5 ms)Closed at 1732131827618
2024-11-20T19:43:47,619 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:47,660 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:47,741 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T19:43:47,742 INFO  [regionserver/e14ef07817f4:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T19:43:47,812 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,45591,1732131801163; all regions closed.
2024-11-20T19:43:47,813 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,813 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,813 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,814 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,814 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:47,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741840_1023 (size=825)
2024-11-20T19:43:47,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741840_1023 (size=825)
2024-11-20T19:43:47,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:47,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:48,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:48,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:49,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:49,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:49,878 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T19:43:50,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:50,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:50,965 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:43:51,410 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta after 4002ms
2024-11-20T19:43:51,412 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/WALs/e14ef07817f4,45591,1732131801163/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta to hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs/e14ef07817f4%2C45591%2C1732131801163.meta.1732131802347.meta
2024-11-20T19:43:51,419 DEBUG [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs
2024-11-20T19:43:51,419 INFO  [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C45591%2C1732131801163.meta:.meta(num 1732131827400)
2024-11-20T19:43:51,419 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,420 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,420 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,420 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,420 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741844_1028 (size=1162)
2024-11-20T19:43:51,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741844_1028 (size=1162)
2024-11-20T19:43:51,426 DEBUG [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs
2024-11-20T19:43:51,426 INFO  [RS:0;e14ef07817f4:45591 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C45591%2C1732131801163:(num 1732131827504)
2024-11-20T19:43:51,426 DEBUG [RS:0;e14ef07817f4:45591 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:43:51,426 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:43:51,426 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:43:51,426 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:43:51,426 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:43:51,426 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:43:51,427 INFO  [RS:0;e14ef07817f4:45591 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45591
2024-11-20T19:43:51,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,45591,1732131801163
2024-11-20T19:43:51,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:43:51,535 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:43:51,598 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,45591,1732131801163]
2024-11-20T19:43:51,608 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,45591,1732131801163 already deleted, retry=false
2024-11-20T19:43:51,608 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,45591,1732131801163 expired; onlineServers=0
2024-11-20T19:43:51,609 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,43153,1732131800986' *****
2024-11-20T19:43:51,609 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:43:51,609 INFO  [M:0;e14ef07817f4:43153 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:43:51,609 INFO  [M:0;e14ef07817f4:43153 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:43:51,610 DEBUG [M:0;e14ef07817f4:43153 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:43:51,610 DEBUG [M:0;e14ef07817f4:43153 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:43:51,610 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:43:51,610 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131801514 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131801514,5,FailOnTimeoutGroup]
2024-11-20T19:43:51,610 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131801514 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131801514,5,FailOnTimeoutGroup]
2024-11-20T19:43:51,611 INFO  [M:0;e14ef07817f4:43153 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:43:51,611 INFO  [M:0;e14ef07817f4:43153 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:43:51,612 DEBUG [M:0;e14ef07817f4:43153 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:43:51,612 INFO  [M:0;e14ef07817f4:43153 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:43:51,612 INFO  [M:0;e14ef07817f4:43153 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:43:51,613 INFO  [M:0;e14ef07817f4:43153 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:43:51,613 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:43:51,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:43:51,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:51,619 DEBUG [M:0;e14ef07817f4:43153 {}] zookeeper.ZKUtil(347): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:43:51,619 WARN  [M:0;e14ef07817f4:43153 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:43:51,620 INFO  [M:0;e14ef07817f4:43153 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/.lastflushedseqids
2024-11-20T19:43:51,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741846_1030 (size=111)
2024-11-20T19:43:51,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741846_1030 (size=111)
2024-11-20T19:43:51,627 INFO  [M:0;e14ef07817f4:43153 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:43:51,627 INFO  [M:0;e14ef07817f4:43153 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:43:51,628 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:43:51,628 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:51,628 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:51,628 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:43:51,628 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:51,628 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-20T19:43:51,628 ERROR [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData-prefix:e14ef07817f4,43153,1732131800986 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:51,628 WARN  [FSHLog-0-hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData-prefix:e14ef07817f4,43153,1732131800986 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:51,628 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog e14ef07817f4%2C43153%2C1732131800986:(num 1732131801303) roll requested
2024-11-20T19:43:51,629 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C43153%2C1732131800986.1732131831629
2024-11-20T19:43:51,635 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,636 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,636 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,636 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,636 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:51,636 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131831629
2024-11-20T19:43:51,636 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:51,636 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34583,DS-52d2ef8f-e12c-40fa-a9e2-c64fe4648c26,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T19:43:51,637 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303
2024-11-20T19:43:51,637 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42673:42673),(127.0.0.1/127.0.0.1:41253:41253)]
2024-11-20T19:43:51,637 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 is not closed yet, will try archiving it next time
2024-11-20T19:43:51,637 WARN  [IPC Server handler 4 on default port 36317 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-20T19:43:51,637 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 after 0ms
2024-11-20T19:43:51,650 DEBUG [M:0;e14ef07817f4:43153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9723fe479b4e27b747c79d40dad70f is 82, key is hbase:meta,,1/info:regioninfo/1732131802378/Put/seqid=0
2024-11-20T19:43:51,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741848_1033 (size=5672)
2024-11-20T19:43:51,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741848_1033 (size=5672)
2024-11-20T19:43:51,655 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9723fe479b4e27b747c79d40dad70f
2024-11-20T19:43:51,673 DEBUG [M:0;e14ef07817f4:43153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9b81862f89d401facd6e835f8183bca is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732131802901/Put/seqid=0
2024-11-20T19:43:51,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741849_1034 (size=6118)
2024-11-20T19:43:51,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741849_1034 (size=6118)
2024-11-20T19:43:51,682 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9b81862f89d401facd6e835f8183bca
2024-11-20T19:43:51,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:51,698 INFO  [RS:0;e14ef07817f4:45591 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:43:51,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45591-0x10159c5c3330001, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:51,698 INFO  [RS:0;e14ef07817f4:45591 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,45591,1732131801163; zookeeper connection closed.
2024-11-20T19:43:51,698 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57584d3b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57584d3b
2024-11-20T19:43:51,698 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:43:51,700 DEBUG [M:0;e14ef07817f4:43153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75f6709e1db748aa9e588adcad64abd6 is 69, key is e14ef07817f4,45591,1732131801163/rs:state/1732131801637/Put/seqid=0
2024-11-20T19:43:51,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741850_1035 (size=5156)
2024-11-20T19:43:51,704 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741850_1035 (size=5156)
2024-11-20T19:43:51,705 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75f6709e1db748aa9e588adcad64abd6
2024-11-20T19:43:51,722 DEBUG [M:0;e14ef07817f4:43153 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f7e70e16644af1bc0de37fb1008762 is 52, key is load_balancer_on/state:d/1732131802506/Put/seqid=0
2024-11-20T19:43:51,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741851_1036 (size=5056)
2024-11-20T19:43:51,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741851_1036 (size=5056)
2024-11-20T19:43:51,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:51,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:52,129 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f7e70e16644af1bc0de37fb1008762
2024-11-20T19:43:52,168 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b9723fe479b4e27b747c79d40dad70f as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b9723fe479b4e27b747c79d40dad70f
2024-11-20T19:43:52,177 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b9723fe479b4e27b747c79d40dad70f, entries=8, sequenceid=56, filesize=5.5 K
2024-11-20T19:43:52,178 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9b81862f89d401facd6e835f8183bca as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9b81862f89d401facd6e835f8183bca
2024-11-20T19:43:52,184 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9b81862f89d401facd6e835f8183bca, entries=6, sequenceid=56, filesize=6.0 K
2024-11-20T19:43:52,185 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/75f6709e1db748aa9e588adcad64abd6 as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75f6709e1db748aa9e588adcad64abd6
2024-11-20T19:43:52,190 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/75f6709e1db748aa9e588adcad64abd6, entries=1, sequenceid=56, filesize=5.0 K
2024-11-20T19:43:52,190 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/71f7e70e16644af1bc0de37fb1008762 as hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/71f7e70e16644af1bc0de37fb1008762
2024-11-20T19:43:52,195 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/71f7e70e16644af1bc0de37fb1008762, entries=1, sequenceid=56, filesize=4.9 K
2024-11-20T19:43:52,196 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 568ms, sequenceid=56, compaction requested=false
2024-11-20T19:43:52,197 INFO  [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:52,198 DEBUG [M:0;e14ef07817f4:43153 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131831628Disabling compacts and flushes for region at 1732131831628Disabling writes for close at 1732131831628Obtaining lock to block concurrent updates at 1732131831628Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131831628Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732131831628Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131831637 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131831637Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131831649 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131831649Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131831659 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131831673 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131831673Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131831687 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131831699 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131831699Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131831709 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131831722 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131831722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66386f7e: reopening flushed file at 1732131832166 (+444 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b924094: reopening flushed file at 1732131832177 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51a534dd: reopening flushed file at 1732131832184 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3589326a: reopening flushed file at 1732131832190 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 568ms, sequenceid=56, compaction requested=false at 1732131832196 (+6 ms)Writing region close event to WAL at 1732131832197 (+1 ms)Closed at 1732131832197
2024-11-20T19:43:52,198 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:52,198 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:52,198 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:52,198 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:52,198 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:43:52,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40215 is added to blk_1073741847_1031 (size=757)
2024-11-20T19:43:52,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741847_1031 (size=757)
2024-11-20T19:43:52,619 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,620 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,640 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,641 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,641 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,641 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,644 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,644 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,644 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,646 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,649 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:52,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:52,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:52,878 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T19:43:53,152 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:43:53,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,157 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,158 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,158 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,178 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,179 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,182 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,184 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:43:53,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:53,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:54,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:54,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:55,638 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 after 4001ms
2024-11-20T19:43:55,639 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/WALs/e14ef07817f4,43153,1732131800986/e14ef07817f4%2C43153%2C1732131800986.1732131801303 to hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/oldWALs/e14ef07817f4%2C43153%2C1732131800986.1732131801303
2024-11-20T19:43:55,646 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/MasterData/oldWALs/e14ef07817f4%2C43153%2C1732131800986.1732131801303 to hdfs://localhost:36317/user/jenkins/test-data/4cc321fc-7cef-d385-b3fa-47897ae469e6/oldWALs/e14ef07817f4%2C43153%2C1732131800986.1732131801303$masterlocalwal$
2024-11-20T19:43:55,646 INFO  [M:0;e14ef07817f4:43153 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:43:55,646 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:43:55,646 INFO  [M:0;e14ef07817f4:43153 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43153
2024-11-20T19:43:55,646 INFO  [M:0;e14ef07817f4:43153 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:43:55,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:55,800 INFO  [M:0;e14ef07817f4:43153 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:43:55,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43153-0x10159c5c3330000, quorum=127.0.0.1:64289, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:43:55,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:55,832 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@649d2ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:55,833 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1b0a2346{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:55,834 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:55,834 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7202b0aa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:55,834 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78f99ac1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:55,838 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:55,838 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:55,838 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:55,838 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid 78cf0a18-0a10-4f53-bc0c-4f4b3f88013b) service to localhost/127.0.0.1:36317
2024-11-20T19:43:55,839 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data3/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:55,839 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data4/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:55,839 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:55,841 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c3c718b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:55,842 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7df90452{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:55,842 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:55,842 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58c81822{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:55,842 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ae70be3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:55,843 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:43:55,843 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:43:55,843 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:43:55,843 WARN  [BP-1075613540-172.17.0.3-1732131798548 heartbeating to localhost/127.0.0.1:36317 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1075613540-172.17.0.3-1732131798548 (Datanode Uuid a89e6e79-7275-47b4-8d5f-af5885cf361d) service to localhost/127.0.0.1:36317
2024-11-20T19:43:55,843 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data1/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:55,844 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/cluster_910c9823-138b-689f-be25-61244a91d1a5/data/data2/current/BP-1075613540-172.17.0.3-1732131798548 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:43:55,844 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:43:55,849 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@482b646b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:43:55,850 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c7a3196{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:43:55,850 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:43:55,850 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42369482{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:43:55,851 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@37ba1ac4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir/,STOPPED}
2024-11-20T19:43:55,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:55,857 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:43:55,876 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:43:55,883 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=180 (was 155)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36317
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36317
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36317 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36317 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36317
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36317 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36317
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:36317
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=35 (was 53), ProcessCount=11 (was 11), AvailableMemoryMB=7328 (was 7415)
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=180, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=35, ProcessCount=11, AvailableMemoryMB=7328
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.log.dir so I do NOT create it in target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/60dc9794-f8ef-97b3-e873-6b88c562b120/hadoop.tmp.dir so I do NOT create it in target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75, deleteOnExit=true
2024-11-20T19:43:55,889 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/test.cache.data in system properties and HBase conf
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:43:55,890 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:43:55,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:43:55,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:43:55,904 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:43:56,306 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:56,310 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:56,311 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:56,312 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:56,312 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:56,312 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:56,312 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2aaa4790{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:56,313 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e470e04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:56,403 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ed77c81{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/java.io.tmpdir/jetty-localhost-40821-hadoop-hdfs-3_4_1-tests_jar-_-any-4205399778845681176/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:43:56,404 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@222377c1{HTTP/1.1, (http/1.1)}{localhost:40821}
2024-11-20T19:43:56,404 INFO  [Time-limited test {}] server.Server(415): Started @195478ms
2024-11-20T19:43:56,415 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:43:56,676 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:56,679 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:56,679 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:56,679 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:56,679 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:43:56,680 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3be31a0b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:56,680 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@443c49f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:56,772 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@457507e6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/java.io.tmpdir/jetty-localhost-40033-hadoop-hdfs-3_4_1-tests_jar-_-any-11030193178828038542/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:56,773 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43d16ee8{HTTP/1.1, (http/1.1)}{localhost:40033}
2024-11-20T19:43:56,773 INFO  [Time-limited test {}] server.Server(415): Started @195847ms
2024-11-20T19:43:56,774 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:56,798 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:43:56,802 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:43:56,802 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:43:56,802 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:43:56,802 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:43:56,803 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1beefc80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:43:56,803 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26731840{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:43:56,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:56,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:56,901 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b0e5884{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/java.io.tmpdir/jetty-localhost-44001-hadoop-hdfs-3_4_1-tests_jar-_-any-13695266926500087287/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:43:56,901 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2526c219{HTTP/1.1, (http/1.1)}{localhost:44001}
2024-11-20T19:43:56,901 INFO  [Time-limited test {}] server.Server(415): Started @195975ms
2024-11-20T19:43:56,903 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:43:57,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:43:57,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:43:57,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T19:43:57,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T19:43:57,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:57,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:58,221 WARN  [Thread-1652 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data2/current/BP-919207744-172.17.0.3-1732131835911/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:58,221 WARN  [Thread-1651 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data1/current/BP-919207744-172.17.0.3-1732131835911/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:58,245 WARN  [Thread-1615 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:58,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x482bc7db93319595 with lease ID 0xc368a028bbad96d5: Processing first storage report for DS-fdc9494e-640e-4a4e-8220-e4043554ca49 from datanode DatanodeRegistration(127.0.0.1:39177, datanodeUuid=e07b774e-4762-4bfc-afa5-e190cd5a3196, infoPort=36407, infoSecurePort=0, ipcPort=36053, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911)
2024-11-20T19:43:58,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x482bc7db93319595 with lease ID 0xc368a028bbad96d5: from storage DS-fdc9494e-640e-4a4e-8220-e4043554ca49 node DatanodeRegistration(127.0.0.1:39177, datanodeUuid=e07b774e-4762-4bfc-afa5-e190cd5a3196, infoPort=36407, infoSecurePort=0, ipcPort=36053, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:58,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x482bc7db93319595 with lease ID 0xc368a028bbad96d5: Processing first storage report for DS-79e97cc3-edd5-4335-a365-cd982017f91c from datanode DatanodeRegistration(127.0.0.1:39177, datanodeUuid=e07b774e-4762-4bfc-afa5-e190cd5a3196, infoPort=36407, infoSecurePort=0, ipcPort=36053, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911)
2024-11-20T19:43:58,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x482bc7db93319595 with lease ID 0xc368a028bbad96d5: from storage DS-79e97cc3-edd5-4335-a365-cd982017f91c node DatanodeRegistration(127.0.0.1:39177, datanodeUuid=e07b774e-4762-4bfc-afa5-e190cd5a3196, infoPort=36407, infoSecurePort=0, ipcPort=36053, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:58,333 WARN  [Thread-1662 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data3/current/BP-919207744-172.17.0.3-1732131835911/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:58,333 WARN  [Thread-1663 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data4/current/BP-919207744-172.17.0.3-1732131835911/current, will proceed with Du for space computation calculation, 
2024-11-20T19:43:58,356 WARN  [Thread-1638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:43:58,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbcbbdced4aa36c10 with lease ID 0xc368a028bbad96d6: Processing first storage report for DS-e4cce607-3bd7-46a4-bb62-4a310ff14931 from datanode DatanodeRegistration(127.0.0.1:41371, datanodeUuid=f27e1872-06d7-4f54-a6ef-89f425aa57f0, infoPort=38369, infoSecurePort=0, ipcPort=45027, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911)
2024-11-20T19:43:58,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcbbdced4aa36c10 with lease ID 0xc368a028bbad96d6: from storage DS-e4cce607-3bd7-46a4-bb62-4a310ff14931 node DatanodeRegistration(127.0.0.1:41371, datanodeUuid=f27e1872-06d7-4f54-a6ef-89f425aa57f0, infoPort=38369, infoSecurePort=0, ipcPort=45027, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:43:58,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbcbbdced4aa36c10 with lease ID 0xc368a028bbad96d6: Processing first storage report for DS-02db6607-a7af-4913-84a3-e4e90dad2835 from datanode DatanodeRegistration(127.0.0.1:41371, datanodeUuid=f27e1872-06d7-4f54-a6ef-89f425aa57f0, infoPort=38369, infoSecurePort=0, ipcPort=45027, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911)
2024-11-20T19:43:58,358 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbcbbdced4aa36c10 with lease ID 0xc368a028bbad96d6: from storage DS-02db6607-a7af-4913-84a3-e4e90dad2835 node DatanodeRegistration(127.0.0.1:41371, datanodeUuid=f27e1872-06d7-4f54-a6ef-89f425aa57f0, infoPort=38369, infoSecurePort=0, ipcPort=45027, storageInfo=lv=-57;cid=testClusterID;nsid=1643760033;c=1732131835911), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:43:58,441 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab
2024-11-20T19:43:58,468 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/zookeeper_0, clientPort=60066, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:43:58,469 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60066
2024-11-20T19:43:58,470 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,472 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:43:58,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:43:58,484 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768 with version=8
2024-11-20T19:43:58,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:43:58,486 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:43:58,486 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,486 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,487 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:43:58,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,487 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:43:58,487 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:43:58,487 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:43:58,488 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38821
2024-11-20T19:43:58,490 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38821 connecting to ZooKeeper ensemble=127.0.0.1:60066
2024-11-20T19:43:58,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388210x0, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:43:58,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38821-0x10159c655af0000 connected
2024-11-20T19:43:58,745 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,750 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:58,755 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768, hbase.cluster.distributed=false
2024-11-20T19:43:58,756 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:43:58,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38821
2024-11-20T19:43:58,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38821
2024-11-20T19:43:58,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38821
2024-11-20T19:43:58,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38821
2024-11-20T19:43:58,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38821
2024-11-20T19:43:58,771 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:43:58,771 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:43:58,772 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:43:58,773 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41837
2024-11-20T19:43:58,774 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41837 connecting to ZooKeeper ensemble=127.0.0.1:60066
2024-11-20T19:43:58,775 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,776 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:418370x0, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:43:58,787 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41837-0x10159c655af0001 connected
2024-11-20T19:43:58,787 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:43:58,787 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:43:58,788 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:43:58,789 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:43:58,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:43:58,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41837
2024-11-20T19:43:58,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41837
2024-11-20T19:43:58,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41837
2024-11-20T19:43:58,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41837
2024-11-20T19:43:58,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41837
2024-11-20T19:43:58,810 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:38821
2024-11-20T19:43:58,811 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,38821,1732131838486
2024-11-20T19:43:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:58,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:58,818 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,38821,1732131838486
2024-11-20T19:43:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:43:58,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:58,829 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:43:58,829 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,38821,1732131838486 from backup master directory
2024-11-20T19:43:58,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:58,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,38821,1732131838486
2024-11-20T19:43:58,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:58,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:43:58,839 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:43:58,839 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,38821,1732131838486
2024-11-20T19:43:58,846 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/hbase.id] with ID: aef1802e-f92b-41dc-89ec-71b14c9d6835
2024-11-20T19:43:58,846 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/.tmp/hbase.id
2024-11-20T19:43:58,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:43:58,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:43:58,854 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/.tmp/hbase.id]:[hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/hbase.id]
2024-11-20T19:43:58,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:58,863 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:43:58,863 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:43:58,865 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T19:43:58,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:58,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:58,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:43:58,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:43:58,887 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:43:58,887 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:43:58,888 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:58,895 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:43:58,895 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:43:58,896 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store
2024-11-20T19:43:58,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:43:58,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:43:58,905 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:58,905 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:43:58,905 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131838905Disabling compacts and flushes for region at 1732131838905Disabling writes for close at 1732131838905Writing region close event to WAL at 1732131838905Closed at 1732131838905
2024-11-20T19:43:58,906 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/.initializing
2024-11-20T19:43:58,906 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/WALs/e14ef07817f4,38821,1732131838486
2024-11-20T19:43:58,909 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C38821%2C1732131838486, suffix=, logDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/WALs/e14ef07817f4,38821,1732131838486, archiveDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/oldWALs, maxLogs=10
2024-11-20T19:43:58,909 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C38821%2C1732131838486.1732131838909
2024-11-20T19:43:58,914 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/WALs/e14ef07817f4,38821,1732131838486/e14ef07817f4%2C38821%2C1732131838486.1732131838909
2024-11-20T19:43:58,917 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38369:38369),(127.0.0.1/127.0.0.1:36407:36407)]
2024-11-20T19:43:58,918 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:43:58,918 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:58,918 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,918 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,919 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:58,921 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:43:58,921 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:58,921 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:58,921 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:58,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:43:58,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:58,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:58,923 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:58,924 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:43:58,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:58,925 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:58,925 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:43:58,926 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:43:58,926 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:58,926 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:43:58,927 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,927 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,927 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,929 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,929 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,929 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:43:58,930 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:43:58,933 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:43:58,933 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812691, jitterRate=0.03339017927646637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:43:58,933 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131838918Initializing all the Stores at 1732131838919 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131838919Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131838919Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131838919Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131838919Cleaning up temporary data from old regions at 1732131838929 (+10 ms)Region opened successfully at 1732131838933 (+4 ms)
2024-11-20T19:43:58,934 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:43:58,936 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d03b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:43:58,937 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:43:58,937 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:43:58,938 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:43:58,938 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:43:58,938 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:43:58,938 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:43:58,938 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:43:58,941 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:43:58,942 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:43:58,955 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:43:58,955 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:43:58,956 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:43:58,965 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:43:58,966 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:43:58,967 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:43:58,976 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:43:58,977 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:43:58,986 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:43:58,988 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:43:58,997 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:43:59,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:59,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:43:59,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,008 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,38821,1732131838486, sessionid=0x10159c655af0000, setting cluster-up flag (Was=false)
2024-11-20T19:43:59,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,060 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:43:59,062 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,38821,1732131838486
2024-11-20T19:43:59,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,113 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:43:59,115 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,38821,1732131838486
2024-11-20T19:43:59,117 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:43:59,119 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:59,120 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:43:59,120 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:43:59,121 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,38821,1732131838486
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:43:59,124 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:43:59,125 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131869126
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:43:59,126 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:43:59,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:43:59,127 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:43:59,127 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:43:59,127 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131839127,5,FailOnTimeoutGroup]
2024-11-20T19:43:59,128 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131839127,5,FailOnTimeoutGroup]
2024-11-20T19:43:59,128 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,128 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:43:59,128 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,128 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,128 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,128 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:43:59,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:43:59,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:43:59,138 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:43:59,138 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768
2024-11-20T19:43:59,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:43:59,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:43:59,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:59,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:43:59,147 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:43:59,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:43:59,150 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:43:59,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:43:59,152 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:43:59,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,153 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:43:59,154 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:43:59,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,155 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:43:59,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740
2024-11-20T19:43:59,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740
2024-11-20T19:43:59,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:43:59,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:43:59,159 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:43:59,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:43:59,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:43:59,164 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881410, jitterRate=0.12077178061008453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:43:59,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131839145Initializing all the Stores at 1732131839146 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839146Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839146Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131839146Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839146Cleaning up temporary data from old regions at 1732131839158 (+12 ms)Region opened successfully at 1732131839165 (+7 ms)
2024-11-20T19:43:59,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:43:59,165 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:43:59,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:43:59,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:43:59,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:43:59,166 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:43:59,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131839165Disabling compacts and flushes for region at 1732131839165Disabling writes for close at 1732131839165Writing region close event to WAL at 1732131839166 (+1 ms)Closed at 1732131839166
2024-11-20T19:43:59,168 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:59,168 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:43:59,168 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:43:59,170 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:43:59,171 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:43:59,195 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(746): ClusterId : aef1802e-f92b-41dc-89ec-71b14c9d6835
2024-11-20T19:43:59,195 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:43:59,210 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:43:59,210 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:43:59,220 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:43:59,221 DEBUG [RS:0;e14ef07817f4:41837 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc84b6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:43:59,238 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:41837
2024-11-20T19:43:59,238 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:43:59,238 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:43:59,238 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:43:59,239 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,38821,1732131838486 with port=41837, startcode=1732131838771
2024-11-20T19:43:59,239 DEBUG [RS:0;e14ef07817f4:41837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:43:59,241 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37101, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:43:59,241 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38821 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,241 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38821 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,243 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768
2024-11-20T19:43:59,243 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32939
2024-11-20T19:43:59,243 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:43:59,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:43:59,250 DEBUG [RS:0;e14ef07817f4:41837 {}] zookeeper.ZKUtil(111): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,250 WARN  [RS:0;e14ef07817f4:41837 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:43:59,250 INFO  [RS:0;e14ef07817f4:41837 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:59,250 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,250 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,41837,1732131838771]
2024-11-20T19:43:59,254 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:43:59,255 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:43:59,255 INFO  [RS:0;e14ef07817f4:41837 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:43:59,255 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,255 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:43:59,256 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:43:59,256 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:43:59,257 DEBUG [RS:0;e14ef07817f4:41837 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:43:59,260 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,260 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,261 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,261 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,261 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,261 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,41837,1732131838771-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:43:59,276 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:43:59,276 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,41837,1732131838771-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,276 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,276 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.Replication(171): e14ef07817f4,41837,1732131838771 started
2024-11-20T19:43:59,289 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,289 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,41837,1732131838771, RpcServer on e14ef07817f4/172.17.0.3:41837, sessionid=0x10159c655af0001
2024-11-20T19:43:59,289 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:43:59,289 DEBUG [RS:0;e14ef07817f4:41837 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,289 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,41837,1732131838771'
2024-11-20T19:43:59,289 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:43:59,290 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:43:59,290 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:43:59,290 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:43:59,290 DEBUG [RS:0;e14ef07817f4:41837 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,290 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,41837,1732131838771'
2024-11-20T19:43:59,291 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:43:59,291 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:43:59,291 DEBUG [RS:0;e14ef07817f4:41837 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:43:59,291 INFO  [RS:0;e14ef07817f4:41837 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:43:59,291 INFO  [RS:0;e14ef07817f4:41837 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:43:59,321 WARN  [e14ef07817f4:38821 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:43:59,393 INFO  [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C41837%2C1732131838771, suffix=, logDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771, archiveDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs, maxLogs=32
2024-11-20T19:43:59,394 INFO  [RS:0;e14ef07817f4:41837 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41837%2C1732131838771.1732131839394
2024-11-20T19:43:59,399 INFO  [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131839394
2024-11-20T19:43:59,400 DEBUG [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38369:38369),(127.0.0.1/127.0.0.1:36407:36407)]
2024-11-20T19:43:59,571 DEBUG [e14ef07817f4:38821 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:43:59,572 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,575 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,41837,1732131838771, state=OPENING
2024-11-20T19:43:59,586 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:43:59,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:43:59,598 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:43:59,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:59,599 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,41837,1732131838771}]
2024-11-20T19:43:59,599 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:59,756 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:43:59,759 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38689, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:43:59,769 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:43:59,769 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:43:59,772 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C41837%2C1732131838771.meta, suffix=.meta, logDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771, archiveDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs, maxLogs=32
2024-11-20T19:43:59,772 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41837%2C1732131838771.meta.1732131839772.meta
2024-11-20T19:43:59,778 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.meta.1732131839772.meta
2024-11-20T19:43:59,780 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36407:36407),(127.0.0.1/127.0.0.1:38369:38369)]
2024-11-20T19:43:59,783 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:43:59,783 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:43:59,783 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:43:59,783 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:43:59,783 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:43:59,783 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:43:59,784 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:43:59,784 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:43:59,785 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:43:59,787 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:43:59,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:43:59,789 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:43:59,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,790 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,790 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:43:59,791 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:43:59,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,792 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,792 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:43:59,793 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:43:59,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:43:59,793 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:43:59,793 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:43:59,794 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740
2024-11-20T19:43:59,795 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740
2024-11-20T19:43:59,796 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:43:59,796 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:43:59,796 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:43:59,797 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:43:59,798 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=857045, jitterRate=0.08978913724422455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:43:59,798 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:43:59,799 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131839784Writing region info on filesystem at 1732131839784Initializing all the Stores at 1732131839785 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839785Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839785Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131839785Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131839785Cleaning up temporary data from old regions at 1732131839796 (+11 ms)Running coprocessor post-open hooks at 1732131839798 (+2 ms)Region opened successfully at 1732131839799 (+1 ms)
2024-11-20T19:43:59,800 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131839755
2024-11-20T19:43:59,802 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:43:59,802 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:43:59,803 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,803 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,41837,1732131838771, state=OPEN
2024-11-20T19:43:59,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:59,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:43:59,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:43:59,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:43:59,924 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,41837,1732131838771
2024-11-20T19:43:59,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:59,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:43:59,931 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:43:59,931 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,41837,1732131838771 in 325 msec
2024-11-20T19:43:59,935 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:43:59,935 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 763 msec
2024-11-20T19:43:59,936 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:43:59,936 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:43:59,938 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:43:59,938 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,41837,1732131838771, seqNum=-1]
2024-11-20T19:43:59,938 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:43:59,939 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32979, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:43:59,946 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 827 msec
2024-11-20T19:43:59,946 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131839946, completionTime=-1
2024-11-20T19:43:59,946 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:43:59,946 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:43:59,947 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131899947
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732131959948
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:38821, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,948 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,950 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:43:59,951 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.112sec
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:43:59,952 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:43:59,954 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:43:59,954 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:43:59,955 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,38821,1732131838486-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:43:59,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680fc8dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:43:59,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,38821,-1 for getting cluster id
2024-11-20T19:43:59,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:43:59,998 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'aef1802e-f92b-41dc-89ec-71b14c9d6835'
2024-11-20T19:43:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:43:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "aef1802e-f92b-41dc-89ec-71b14c9d6835"

2024-11-20T19:44:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b78a954, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:44:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,38821,-1]
2024-11-20T19:44:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:44:00,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:00,003 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:44:00,004 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ff6bde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:44:00,005 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:44:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,41837,1732131838771, seqNum=-1]
2024-11-20T19:44:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:44:00,007 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:44:00,009 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,38821,1732131838486
2024-11-20T19:44:00,010 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:00,012 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:44:00,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T19:44:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is e14ef07817f4,38821,1732131838486
2024-11-20T19:44:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ac02d0a
2024-11-20T19:44:00,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T19:44:00,015 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T19:44:00,015 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T19:44:00,015 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T19:44:00,016 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:44:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:00,018 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T19:44:00,018 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:00,018 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-20T19:44:00,020 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T19:44:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:44:00,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741835_1011 (size=405)
2024-11-20T19:44:00,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741835_1011 (size=405)
2024-11-20T19:44:00,027 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7e1c1b494c7c2e15480f0302d97a8c0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768
2024-11-20T19:44:00,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741836_1012 (size=88)
2024-11-20T19:44:00,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741836_1012 (size=88)
2024-11-20T19:44:00,033 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:00,033 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing e7e1c1b494c7c2e15480f0302d97a8c0, disabling compactions & flushes
2024-11-20T19:44:00,034 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,034 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,034 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0. after waiting 0 ms
2024-11-20T19:44:00,034 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,034 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,034 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7e1c1b494c7c2e15480f0302d97a8c0:
Waiting for close lock at 1732131840033Disabling compacts and flushes for region at 1732131840033Disabling writes for close at 1732131840034 (+1 ms)Writing region close event to WAL at 1732131840034Closed at 1732131840034
2024-11-20T19:44:00,035 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T19:44:00,035 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732131840035"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131840035"}]},"ts":"1732131840035"}
2024-11-20T19:44:00,037 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T19:44:00,038 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T19:44:00,038 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131840038"}]},"ts":"1732131840038"}
2024-11-20T19:44:00,040 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-20T19:44:00,040 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e7e1c1b494c7c2e15480f0302d97a8c0, ASSIGN}]
2024-11-20T19:44:00,041 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e7e1c1b494c7c2e15480f0302d97a8c0, ASSIGN
2024-11-20T19:44:00,042 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e7e1c1b494c7c2e15480f0302d97a8c0, ASSIGN; state=OFFLINE, location=e14ef07817f4,41837,1732131838771; forceNewPlan=false, retain=false
2024-11-20T19:44:00,193 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7e1c1b494c7c2e15480f0302d97a8c0, regionState=OPENING, regionLocation=e14ef07817f4,41837,1732131838771
2024-11-20T19:44:00,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e7e1c1b494c7c2e15480f0302d97a8c0, ASSIGN because future has completed
2024-11-20T19:44:00,201 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7e1c1b494c7c2e15480f0302d97a8c0, server=e14ef07817f4,41837,1732131838771}]
2024-11-20T19:44:00,366 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,366 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e7e1c1b494c7c2e15480f0302d97a8c0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:44:00,367 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,367 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:00,367 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,367 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,370 INFO  [StoreOpener-e7e1c1b494c7c2e15480f0302d97a8c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e7e1c1b494c7c2e15480f0302d97a8c0 
2024-11-20T19:44:00,371 INFO  [StoreOpener-e7e1c1b494c7c2e15480f0302d97a8c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7e1c1b494c7c2e15480f0302d97a8c0 columnFamilyName info
2024-11-20T19:44:00,372 DEBUG [StoreOpener-e7e1c1b494c7c2e15480f0302d97a8c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:00,372 INFO  [StoreOpener-e7e1c1b494c7c2e15480f0302d97a8c0-1 {}] regionserver.HStore(327): Store=e7e1c1b494c7c2e15480f0302d97a8c0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:44:00,372 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,373 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,373 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,374 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,374 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,375 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,377 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:44:00,377 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e7e1c1b494c7c2e15480f0302d97a8c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876390, jitterRate=0.11438851058483124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:44:00,377 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:00,378 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e7e1c1b494c7c2e15480f0302d97a8c0:
Running coprocessor pre-open hook at 1732131840368Writing region info on filesystem at 1732131840368Initializing all the Stores at 1732131840369 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131840369Cleaning up temporary data from old regions at 1732131840374 (+5 ms)Running coprocessor post-open hooks at 1732131840377 (+3 ms)Region opened successfully at 1732131840378 (+1 ms)
2024-11-20T19:44:00,379 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0., pid=6, masterSystemTime=1732131840357
2024-11-20T19:44:00,381 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,381 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:00,382 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e7e1c1b494c7c2e15480f0302d97a8c0, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,41837,1732131838771
2024-11-20T19:44:00,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7e1c1b494c7c2e15480f0302d97a8c0, server=e14ef07817f4,41837,1732131838771 because future has completed
2024-11-20T19:44:00,388 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T19:44:00,388 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e7e1c1b494c7c2e15480f0302d97a8c0, server=e14ef07817f4,41837,1732131838771 in 184 msec
2024-11-20T19:44:00,391 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T19:44:00,391 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e7e1c1b494c7c2e15480f0302d97a8c0, ASSIGN in 348 msec
2024-11-20T19:44:00,392 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T19:44:00,392 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131840392"}]},"ts":"1732131840392"}
2024-11-20T19:44:00,394 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-20T19:44:00,395 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T19:44:00,397 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 379 msec
2024-11-20T19:44:00,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:00,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:01,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:01,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:02,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:02,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:02,879 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:44:02,882 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,905 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,905 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,905 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:02,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:03,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:03,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:04,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:04,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:05,254 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:44:05,256 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-20T19:44:05,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:05,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:06,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:06,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:07,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T19:44:07,373 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T19:44:07,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:44:07,374 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T19:44:07,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T19:44:07,375 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-20T19:44:07,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:07,375 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T19:44:07,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:07,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:08,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:08,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:09,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:09,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:44:10,038 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T19:44:10,038 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-20T19:44:10,044 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:10,044 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:10,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0., hostname=e14ef07817f4,41837,1732131838771, seqNum=2]
2024-11-20T19:44:10,054 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:10,059 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T19:44:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-20T19:44:10,061 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T19:44:10,062 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T19:44:10,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-20T19:44:10,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:10,227 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e7e1c1b494c7c2e15480f0302d97a8c0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T19:44:10,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/1f30864550eb4b1b92ea4d125895f37a is 1080, key is row0001/info:/1732131850049/Put/seqid=0
2024-11-20T19:44:10,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741837_1013 (size=6033)
2024-11-20T19:44:10,253 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741837_1013 (size=6033)
2024-11-20T19:44:10,254 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/1f30864550eb4b1b92ea4d125895f37a
2024-11-20T19:44:10,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/1f30864550eb4b1b92ea4d125895f37a as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a
2024-11-20T19:44:10,265 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a, entries=1, sequenceid=5, filesize=5.9 K
2024-11-20T19:44:10,266 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 39ms, sequenceid=5, compaction requested=false
2024-11-20T19:44:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e7e1c1b494c7c2e15480f0302d97a8c0:

2024-11-20T19:44:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:10,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-20T19:44:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-20T19:44:10,274 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-20T19:44:10,274 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec
2024-11-20T19:44:10,276 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 220 msec
2024-11-20T19:44:10,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:10,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:11,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:11,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:12,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:12,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:13,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:13,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:14,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:14,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:15,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:15,872 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:16,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:16,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:17,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:17,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:18,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:18,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:19,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:19,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476 after 68084ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:44:19,877 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:19,877 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta after 68085ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T19:44:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-20T19:44:20,108 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T19:44:20,116 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-20T19:44:20,121 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T19:44:20,121 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T19:44:20,121 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T19:44:20,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-20T19:44:20,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:20,276 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e7e1c1b494c7c2e15480f0302d97a8c0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T19:44:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f is 1080, key is row0002/info:/1732131860111/Put/seqid=0
2024-11-20T19:44:20,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741838_1014 (size=6033)
2024-11-20T19:44:20,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741838_1014 (size=6033)
2024-11-20T19:44:20,293 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f
2024-11-20T19:44:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f
2024-11-20T19:44:20,306 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f, entries=1, sequenceid=9, filesize=5.9 K
2024-11-20T19:44:20,307 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 31ms, sequenceid=9, compaction requested=false
2024-11-20T19:44:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e7e1c1b494c7c2e15480f0302d97a8c0:

2024-11-20T19:44:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-20T19:44:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-20T19:44:20,312 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-20T19:44:20,312 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 189 msec
2024-11-20T19:44:20,315 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 197 msec
2024-11-20T19:44:20,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:20,878 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:21,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:21,879 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:22,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:22,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:23,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:23,881 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:24,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:24,882 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:25,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:25,884 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:26,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:26,885 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:27,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:27,886 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:28,441 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:44:28,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:28,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:29,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:29,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-20T19:44:30,167 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T19:44:30,174 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41837%2C1732131838771.1732131870174
2024-11-20T19:44:30,235 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:30,235 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:30,235 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:30,236 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:30,236 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:30,236 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131839394 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131870174
2024-11-20T19:44:30,237 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36407:36407),(127.0.0.1/127.0.0.1:38369:38369)]
2024-11-20T19:44:30,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131839394 is not closed yet, will try archiving it next time
2024-11-20T19:44:30,239 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:30,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741833_1009 (size=5546)
2024-11-20T19:44:30,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741833_1009 (size=5546)
2024-11-20T19:44:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-20T19:44:30,243 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T19:44:30,244 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T19:44:30,244 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T19:44:30,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-20T19:44:30,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:30,401 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e7e1c1b494c7c2e15480f0302d97a8c0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T19:44:30,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/cb648164e14f4b03b888bba6a182ffd6 is 1080, key is row0003/info:/1732131870171/Put/seqid=0
2024-11-20T19:44:30,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741840_1016 (size=6033)
2024-11-20T19:44:30,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741840_1016 (size=6033)
2024-11-20T19:44:30,418 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/cb648164e14f4b03b888bba6a182ffd6
2024-11-20T19:44:30,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/cb648164e14f4b03b888bba6a182ffd6 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6
2024-11-20T19:44:30,431 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6, entries=1, sequenceid=13, filesize=5.9 K
2024-11-20T19:44:30,432 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 31ms, sequenceid=13, compaction requested=true
2024-11-20T19:44:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e7e1c1b494c7c2e15480f0302d97a8c0:

2024-11-20T19:44:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:30,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-20T19:44:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-20T19:44:30,436 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-20T19:44:30,436 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec
2024-11-20T19:44:30,439 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 198 msec
2024-11-20T19:44:30,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:30,889 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:31,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:31,891 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:32,876 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:32,892 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:33,878 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:33,893 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:34,879 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:34,894 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:35,880 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:35,895 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:36,882 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:36,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:37,883 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:37,897 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:38,885 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:38,898 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:39,886 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:39,899 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:39,955 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T19:44:39,955 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T19:44:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-20T19:44:40,327 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T19:44:40,328 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:44:40,331 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:44:40,332 DEBUG [Time-limited test {}] regionserver.HStore(1541): e7e1c1b494c7c2e15480f0302d97a8c0/info is initiating minor compaction (all files)
2024-11-20T19:44:40,332 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:44:40,332 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:40,332 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e7e1c1b494c7c2e15480f0302d97a8c0/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:40,333 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6] into tmpdir=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp, totalSize=17.7 K
2024-11-20T19:44:40,334 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1f30864550eb4b1b92ea4d125895f37a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732131850049
2024-11-20T19:44:40,335 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e0dcb87ee7dd4aff8e3bfd7e9ab6a90f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732131860111
2024-11-20T19:44:40,336 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb648164e14f4b03b888bba6a182ffd6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732131870171
2024-11-20T19:44:40,351 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): e7e1c1b494c7c2e15480f0302d97a8c0#info#compaction#45 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:44:40,352 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/8f054d41f3b947228adb3127402423bf is 1080, key is row0001/info:/1732131850049/Put/seqid=0
2024-11-20T19:44:40,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741841_1017 (size=8296)
2024-11-20T19:44:40,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741841_1017 (size=8296)
2024-11-20T19:44:40,365 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/8f054d41f3b947228adb3127402423bf as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/8f054d41f3b947228adb3127402423bf
2024-11-20T19:44:40,371 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e7e1c1b494c7c2e15480f0302d97a8c0/info of e7e1c1b494c7c2e15480f0302d97a8c0 into 8f054d41f3b947228adb3127402423bf(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:44:40,371 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e7e1c1b494c7c2e15480f0302d97a8c0:

2024-11-20T19:44:40,374 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41837%2C1732131838771.1732131880373
2024-11-20T19:44:40,379 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:40,379 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:40,379 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:40,379 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:40,380 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:40,380 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131870174 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131880373
2024-11-20T19:44:40,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741839_1015 (size=2520)
2024-11-20T19:44:40,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741839_1015 (size=2520)
2024-11-20T19:44:40,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38369:38369),(127.0.0.1/127.0.0.1:36407:36407)]
2024-11-20T19:44:40,384 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131839394 to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs/e14ef07817f4%2C41837%2C1732131838771.1732131839394
2024-11-20T19:44:40,385 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-20T19:44:40,387 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T19:44:40,388 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T19:44:40,388 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T19:44:40,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41837 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-20T19:44:40,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:40,542 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e7e1c1b494c7c2e15480f0302d97a8c0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T19:44:40,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/9f058e9a325946a7af3df03cba2e1ede is 1080, key is row0000/info:/1732131880372/Put/seqid=0
2024-11-20T19:44:40,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741843_1019 (size=6033)
2024-11-20T19:44:40,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741843_1019 (size=6033)
2024-11-20T19:44:40,560 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/9f058e9a325946a7af3df03cba2e1ede
2024-11-20T19:44:40,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/9f058e9a325946a7af3df03cba2e1ede as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/9f058e9a325946a7af3df03cba2e1ede
2024-11-20T19:44:40,573 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/9f058e9a325946a7af3df03cba2e1ede, entries=1, sequenceid=18, filesize=5.9 K
2024-11-20T19:44:40,574 INFO  [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 32ms, sequenceid=18, compaction requested=false
2024-11-20T19:44:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e7e1c1b494c7c2e15480f0302d97a8c0:

2024-11-20T19:44:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:40,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-20T19:44:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-20T19:44:40,579 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-20T19:44:40,579 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-11-20T19:44:40,581 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-11-20T19:44:40,887 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:40,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:41,888 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:41,901 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:42,890 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:42,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:43,891 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:43,904 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:44,892 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:44,905 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:45,367 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e7e1c1b494c7c2e15480f0302d97a8c0, had cached 0 bytes from a total of 14329
2024-11-20T19:44:45,893 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:45,906 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:46,894 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:46,907 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:47,896 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:47,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:48,897 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:48,909 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:49,898 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:49,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38821 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-20T19:44:50,408 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T19:44:50,416 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C41837%2C1732131838771.1732131890415
2024-11-20T19:44:50,426 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,426 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,427 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,427 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,427 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,427 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131880373 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131890415
2024-11-20T19:44:50,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36407:36407),(127.0.0.1/127.0.0.1:38369:38369)]
2024-11-20T19:44:50,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131880373 is not closed yet, will try archiving it next time
2024-11-20T19:44:50,428 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/WALs/e14ef07817f4,41837,1732131838771/e14ef07817f4%2C41837%2C1732131838771.1732131870174 to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs/e14ef07817f4%2C41837%2C1732131838771.1732131870174
2024-11-20T19:44:50,428 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:44:50,428 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:44:50,428 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:44:50,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:50,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741842_1018 (size=2026)
2024-11-20T19:44:50,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741842_1018 (size=2026)
2024-11-20T19:44:50,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:50,429 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:44:50,429 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:44:50,429 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=951223209, stopped=false
2024-11-20T19:44:50,429 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,38821,1732131838486
2024-11-20T19:44:50,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:44:50,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:44:50,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:50,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:50,506 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:44:50,506 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:44:50,507 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:44:50,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:50,507 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:44:50,507 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:44:50,508 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,41837,1732131838771' *****
2024-11-20T19:44:50,508 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:44:50,508 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:44:50,509 INFO  [RS:0;e14ef07817f4:41837 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:44:50,509 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:44:50,509 INFO  [RS:0;e14ef07817f4:41837 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:44:50,509 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(3091): Received CLOSE for e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:50,510 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,41837,1732131838771
2024-11-20T19:44:50,510 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:44:50,510 INFO  [RS:0;e14ef07817f4:41837 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:41837.
2024-11-20T19:44:50,510 DEBUG [RS:0;e14ef07817f4:41837 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:44:50,510 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7e1c1b494c7c2e15480f0302d97a8c0, disabling compactions & flushes
2024-11-20T19:44:50,511 DEBUG [RS:0;e14ef07817f4:41837 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:50,511 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:50,511 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:50,511 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:44:50,511 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:44:50,511 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:44:50,511 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0. after waiting 0 ms
2024-11-20T19:44:50,511 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:50,511 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:44:50,511 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e7e1c1b494c7c2e15480f0302d97a8c0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T19:44:50,511 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T19:44:50,511 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1325): Online Regions={e7e1c1b494c7c2e15480f0302d97a8c0=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:44:50,511 DEBUG [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e7e1c1b494c7c2e15480f0302d97a8c0
2024-11-20T19:44:50,511 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:44:50,512 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:44:50,512 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:44:50,512 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:44:50,512 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:44:50,512 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-20T19:44:50,516 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/fd5627e2df2a41e6a571d2897ec00053 is 1080, key is row0001/info:/1732131890411/Put/seqid=0
2024-11-20T19:44:50,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741845_1021 (size=6033)
2024-11-20T19:44:50,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741845_1021 (size=6033)
2024-11-20T19:44:50,525 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/fd5627e2df2a41e6a571d2897ec00053
2024-11-20T19:44:50,531 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/.tmp/info/fd5627e2df2a41e6a571d2897ec00053 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/fd5627e2df2a41e6a571d2897ec00053
2024-11-20T19:44:50,534 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/info/ae9908fbd1bb499ba8f2247021c94767 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0./info:regioninfo/1732131840382/Put/seqid=0
2024-11-20T19:44:50,537 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/fd5627e2df2a41e6a571d2897ec00053, entries=1, sequenceid=22, filesize=5.9 K
2024-11-20T19:44:50,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741846_1022 (size=7308)
2024-11-20T19:44:50,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741846_1022 (size=7308)
2024-11-20T19:44:50,538 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 27ms, sequenceid=22, compaction requested=true
2024-11-20T19:44:50,539 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6] to archive
2024-11-20T19:44:50,539 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/info/ae9908fbd1bb499ba8f2247021c94767
2024-11-20T19:44:50,540 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:44:50,541 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/1f30864550eb4b1b92ea4d125895f37a
2024-11-20T19:44:50,542 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/e0dcb87ee7dd4aff8e3bfd7e9ab6a90f
2024-11-20T19:44:50,544 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6 to hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/info/cb648164e14f4b03b888bba6a182ffd6
2024-11-20T19:44:50,544 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e14ef07817f4:38821 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-20T19:44:50,544 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1f30864550eb4b1b92ea4d125895f37a=6033, e0dcb87ee7dd4aff8e3bfd7e9ab6a90f=6033, cb648164e14f4b03b888bba6a182ffd6=6033]
2024-11-20T19:44:50,548 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e7e1c1b494c7c2e15480f0302d97a8c0/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-20T19:44:50,548 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:50,548 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7e1c1b494c7c2e15480f0302d97a8c0:
Waiting for close lock at 1732131890510Running coprocessor pre-close hooks at 1732131890510Disabling compacts and flushes for region at 1732131890510Disabling writes for close at 1732131890511 (+1 ms)Obtaining lock to block concurrent updates at 1732131890511Preparing flush snapshotting stores in e7e1c1b494c7c2e15480f0302d97a8c0 at 1732131890511Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732131890512 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0. at 1732131890512Flushing e7e1c1b494c7c2e15480f0302d97a8c0/info: creating writer at 1732131890512Flushing e7e1c1b494c7c2e15480f0302d97a8c0/info: appending metadata at 1732131890516 (+4 ms)Flushing e7e1c1b494c7c2e15480f0302d97a8c0/info: closing flushed file at 1732131890516Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e739403: reopening flushed file at 1732131890531 (+15 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e7e1c1b494c7c2e15480f0302d97a8c0 in 27ms, sequenceid=22, compaction requested=true at 1732131890538 (+7 ms)Writing region close event to WAL at 1732131890545 (+7 ms)Running coprocessor post-close hooks at 1732131890548 (+3 ms)Closed at 1732131890548
2024-11-20T19:44:50,548 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732131840015.e7e1c1b494c7c2e15480f0302d97a8c0.
2024-11-20T19:44:50,557 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/ns/bfe2f3e04217432d96032cdc6e207d6b is 43, key is default/ns:d/1732131839940/Put/seqid=0
2024-11-20T19:44:50,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741847_1023 (size=5153)
2024-11-20T19:44:50,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741847_1023 (size=5153)
2024-11-20T19:44:50,562 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/ns/bfe2f3e04217432d96032cdc6e207d6b
2024-11-20T19:44:50,578 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/table/a2b1b8c34e134bfd8eb079e337c53380 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732131840392/Put/seqid=0
2024-11-20T19:44:50,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741848_1024 (size=5508)
2024-11-20T19:44:50,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741848_1024 (size=5508)
2024-11-20T19:44:50,583 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/table/a2b1b8c34e134bfd8eb079e337c53380
2024-11-20T19:44:50,589 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/info/ae9908fbd1bb499ba8f2247021c94767 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/info/ae9908fbd1bb499ba8f2247021c94767
2024-11-20T19:44:50,595 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/info/ae9908fbd1bb499ba8f2247021c94767, entries=10, sequenceid=11, filesize=7.1 K
2024-11-20T19:44:50,596 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/ns/bfe2f3e04217432d96032cdc6e207d6b as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/ns/bfe2f3e04217432d96032cdc6e207d6b
2024-11-20T19:44:50,601 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/ns/bfe2f3e04217432d96032cdc6e207d6b, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T19:44:50,602 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/.tmp/table/a2b1b8c34e134bfd8eb079e337c53380 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/table/a2b1b8c34e134bfd8eb079e337c53380
2024-11-20T19:44:50,606 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/table/a2b1b8c34e134bfd8eb079e337c53380, entries=2, sequenceid=11, filesize=5.4 K
2024-11-20T19:44:50,607 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=11, compaction requested=false
2024-11-20T19:44:50,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T19:44:50,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:44:50,612 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:44:50,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131890511Running coprocessor pre-close hooks at 1732131890511Disabling compacts and flushes for region at 1732131890511Disabling writes for close at 1732131890512 (+1 ms)Obtaining lock to block concurrent updates at 1732131890512Preparing flush snapshotting stores in 1588230740 at 1732131890512Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732131890512Flushing stores of hbase:meta,,1.1588230740 at 1732131890513 (+1 ms)Flushing 1588230740/info: creating writer at 1732131890513Flushing 1588230740/info: appending metadata at 1732131890533 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732131890533Flushing 1588230740/ns: creating writer at 1732131890544 (+11 ms)Flushing 1588230740/ns: appending metadata at 1732131890557 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732131890557Flushing 1588230740/table: creating writer at 1732131890566 (+9 ms)Flushing 1588230740/table: appending metadata at 1732131890578 (+12 ms)Flushing 1588230740/table: closing flushed file at 1732131890578Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@675554e4: reopening flushed file at 1732131890588 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aaa32cd: reopening flushed file at 1732131890595 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e056695: reopening flushed file at 1732131890601 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=11, compaction requested=false at 1732131890607 (+6 ms)Writing region close event to WAL at 1732131890608 (+1 ms)Running coprocessor post-close hooks at 1732131890612 (+4 ms)Closed at 1732131890612
2024-11-20T19:44:50,612 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:44:50,712 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,41837,1732131838771; all regions closed.
2024-11-20T19:44:50,713 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,713 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,713 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,713 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,713 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741834_1010 (size=3306)
2024-11-20T19:44:50,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741834_1010 (size=3306)
2024-11-20T19:44:50,721 DEBUG [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs
2024-11-20T19:44:50,721 INFO  [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C41837%2C1732131838771.meta:.meta(num 1732131839772)
2024-11-20T19:44:50,722 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,722 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,722 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,723 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,723 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741844_1020 (size=1252)
2024-11-20T19:44:50,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741844_1020 (size=1252)
2024-11-20T19:44:50,730 DEBUG [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/oldWALs
2024-11-20T19:44:50,730 INFO  [RS:0;e14ef07817f4:41837 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C41837%2C1732131838771:(num 1732131890415)
2024-11-20T19:44:50,730 DEBUG [RS:0;e14ef07817f4:41837 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:50,730 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:44:50,730 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:44:50,731 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:44:50,731 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:44:50,731 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:44:50,731 INFO  [RS:0;e14ef07817f4:41837 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41837
2024-11-20T19:44:50,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,41837,1732131838771
2024-11-20T19:44:50,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:44:50,742 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:44:50,752 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,41837,1732131838771]
2024-11-20T19:44:50,763 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,41837,1732131838771 already deleted, retry=false
2024-11-20T19:44:50,763 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,41837,1732131838771 expired; onlineServers=0
2024-11-20T19:44:50,763 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,38821,1732131838486' *****
2024-11-20T19:44:50,763 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:44:50,763 INFO  [M:0;e14ef07817f4:38821 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:44:50,763 INFO  [M:0;e14ef07817f4:38821 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:44:50,763 DEBUG [M:0;e14ef07817f4:38821 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:44:50,764 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:44:50,764 DEBUG [M:0;e14ef07817f4:38821 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:44:50,764 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131839127 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131839127,5,FailOnTimeoutGroup]
2024-11-20T19:44:50,764 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131839127 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131839127,5,FailOnTimeoutGroup]
2024-11-20T19:44:50,764 INFO  [M:0;e14ef07817f4:38821 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:44:50,764 INFO  [M:0;e14ef07817f4:38821 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:44:50,764 DEBUG [M:0;e14ef07817f4:38821 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:44:50,764 INFO  [M:0;e14ef07817f4:38821 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:44:50,764 INFO  [M:0;e14ef07817f4:38821 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:44:50,765 INFO  [M:0;e14ef07817f4:38821 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:44:50,765 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:44:50,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:44:50,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:50,773 DEBUG [M:0;e14ef07817f4:38821 {}] zookeeper.ZKUtil(347): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:44:50,773 WARN  [M:0;e14ef07817f4:38821 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:44:50,774 INFO  [M:0;e14ef07817f4:38821 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/.lastflushedseqids
2024-11-20T19:44:50,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741849_1025 (size=130)
2024-11-20T19:44:50,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741849_1025 (size=130)
2024-11-20T19:44:50,782 INFO  [M:0;e14ef07817f4:38821 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:44:50,782 INFO  [M:0;e14ef07817f4:38821 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:44:50,782 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:44:50,782 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:50,782 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:50,783 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:44:50,783 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:50,783 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.61 KB heapSize=55.02 KB
2024-11-20T19:44:50,801 DEBUG [M:0;e14ef07817f4:38821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/556c5ec27fd8494dbc0a9410e22b59cf is 82, key is hbase:meta,,1/info:regioninfo/1732131839803/Put/seqid=0
2024-11-20T19:44:50,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741850_1026 (size=5672)
2024-11-20T19:44:50,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741850_1026 (size=5672)
2024-11-20T19:44:50,806 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/556c5ec27fd8494dbc0a9410e22b59cf
2024-11-20T19:44:50,824 DEBUG [M:0;e14ef07817f4:38821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aeede4ff56294bf5ac3f25df1de5ef82 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732131840397/Put/seqid=0
2024-11-20T19:44:50,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741851_1027 (size=7825)
2024-11-20T19:44:50,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741851_1027 (size=7825)
2024-11-20T19:44:50,829 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.01 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aeede4ff56294bf5ac3f25df1de5ef82
2024-11-20T19:44:50,833 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aeede4ff56294bf5ac3f25df1de5ef82
2024-11-20T19:44:50,846 DEBUG [M:0;e14ef07817f4:38821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a042178e029450e9ee7911ec293dd67 is 69, key is e14ef07817f4,41837,1732131838771/rs:state/1732131839242/Put/seqid=0
2024-11-20T19:44:50,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741852_1028 (size=5156)
2024-11-20T19:44:50,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741852_1028 (size=5156)
2024-11-20T19:44:50,851 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a042178e029450e9ee7911ec293dd67
2024-11-20T19:44:50,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:44:50,852 INFO  [RS:0;e14ef07817f4:41837 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:44:50,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41837-0x10159c655af0001, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:44:50,852 INFO  [RS:0;e14ef07817f4:41837 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,41837,1732131838771; zookeeper connection closed.
2024-11-20T19:44:50,853 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b0804d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b0804d
2024-11-20T19:44:50,853 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:44:50,868 DEBUG [M:0;e14ef07817f4:38821 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8ceb23eabf5b4464a05379c2809118a9 is 52, key is load_balancer_on/state:d/1732131840011/Put/seqid=0
2024-11-20T19:44:50,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741853_1029 (size=5056)
2024-11-20T19:44:50,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741853_1029 (size=5056)
2024-11-20T19:44:50,873 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8ceb23eabf5b4464a05379c2809118a9
2024-11-20T19:44:50,878 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/556c5ec27fd8494dbc0a9410e22b59cf as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/556c5ec27fd8494dbc0a9410e22b59cf
2024-11-20T19:44:50,883 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/556c5ec27fd8494dbc0a9410e22b59cf, entries=8, sequenceid=121, filesize=5.5 K
2024-11-20T19:44:50,883 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aeede4ff56294bf5ac3f25df1de5ef82 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aeede4ff56294bf5ac3f25df1de5ef82
2024-11-20T19:44:50,888 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aeede4ff56294bf5ac3f25df1de5ef82
2024-11-20T19:44:50,888 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aeede4ff56294bf5ac3f25df1de5ef82, entries=14, sequenceid=121, filesize=7.6 K
2024-11-20T19:44:50,889 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a042178e029450e9ee7911ec293dd67 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a042178e029450e9ee7911ec293dd67
2024-11-20T19:44:50,894 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a042178e029450e9ee7911ec293dd67, entries=1, sequenceid=121, filesize=5.0 K
2024-11-20T19:44:50,895 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8ceb23eabf5b4464a05379c2809118a9 as hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8ceb23eabf5b4464a05379c2809118a9
2024-11-20T19:44:50,899 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32939/user/jenkins/test-data/989da5b7-970c-8ae7-decc-8a0e20797768/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8ceb23eabf5b4464a05379c2809118a9, entries=1, sequenceid=121, filesize=4.9 K
2024-11-20T19:44:50,899 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:50,900 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false
2024-11-20T19:44:50,901 INFO  [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:50,901 DEBUG [M:0;e14ef07817f4:38821 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131890782Disabling compacts and flushes for region at 1732131890782Disabling writes for close at 1732131890783 (+1 ms)Obtaining lock to block concurrent updates at 1732131890783Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131890783Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44659, getHeapSize=56272, getOffHeapSize=0, getCellsCount=140 at 1732131890783Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131890784 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131890784Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131890801 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131890801Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131890810 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131890823 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131890823Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131890833 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131890845 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131890845Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131890855 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131890868 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131890868Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2530b6d: reopening flushed file at 1732131890877 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66872315: reopening flushed file at 1732131890883 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1038b363: reopening flushed file at 1732131890889 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16c5107e: reopening flushed file at 1732131890894 (+5 ms)Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=121, compaction requested=false at 1732131890900 (+6 ms)Writing region close event to WAL at 1732131890901 (+1 ms)Closed at 1732131890901
2024-11-20T19:44:50,901 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,901 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,901 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,901 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,901 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:44:50,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39177 is added to blk_1073741830_1006 (size=53056)
2024-11-20T19:44:50,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41371 is added to blk_1073741830_1006 (size=53056)
2024-11-20T19:44:50,904 INFO  [M:0;e14ef07817f4:38821 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:44:50,904 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:44:50,904 INFO  [M:0;e14ef07817f4:38821 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38821
2024-11-20T19:44:50,904 INFO  [M:0;e14ef07817f4:38821 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:44:50,911 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:44:51,016 INFO  [M:0;e14ef07817f4:38821 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:44:51,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38821-0x10159c655af0000, quorum=127.0.0.1:60066, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:44:51,018 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b0e5884{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:44:51,019 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2526c219{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:44:51,019 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:44:51,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26731840{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:44:51,019 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1beefc80{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,STOPPED}
2024-11-20T19:44:51,021 WARN  [BP-919207744-172.17.0.3-1732131835911 heartbeating to localhost/127.0.0.1:32939 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:44:51,021 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:44:51,021 WARN  [BP-919207744-172.17.0.3-1732131835911 heartbeating to localhost/127.0.0.1:32939 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-919207744-172.17.0.3-1732131835911 (Datanode Uuid f27e1872-06d7-4f54-a6ef-89f425aa57f0) service to localhost/127.0.0.1:32939
2024-11-20T19:44:51,021 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:44:51,021 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data3/current/BP-919207744-172.17.0.3-1732131835911 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:44:51,022 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data4/current/BP-919207744-172.17.0.3-1732131835911 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:44:51,022 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:44:51,025 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@457507e6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:44:51,026 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43d16ee8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:44:51,026 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:44:51,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@443c49f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:44:51,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3be31a0b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,STOPPED}
2024-11-20T19:44:51,027 WARN  [BP-919207744-172.17.0.3-1732131835911 heartbeating to localhost/127.0.0.1:32939 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:44:51,027 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:44:51,027 WARN  [BP-919207744-172.17.0.3-1732131835911 heartbeating to localhost/127.0.0.1:32939 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-919207744-172.17.0.3-1732131835911 (Datanode Uuid e07b774e-4762-4bfc-afa5-e190cd5a3196) service to localhost/127.0.0.1:32939
2024-11-20T19:44:51,027 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:44:51,028 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data1/current/BP-919207744-172.17.0.3-1732131835911 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:44:51,028 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/cluster_1f7273cf-3688-f2c2-52ce-c63d82f80f75/data/data2/current/BP-919207744-172.17.0.3-1732131835911 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:44:51,028 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:44:51,034 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ed77c81{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:44:51,035 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@222377c1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:44:51,035 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:44:51,035 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e470e04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:44:51,035 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2aaa4790{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir/,STOPPED}
2024-11-20T19:44:51,041 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:44:51,059 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:44:51,066 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 180)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32939 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:32939
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32939
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32939 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32939
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:32939
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32939
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32939 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=65 (was 35) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7268 (was 7328)
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=65, ProcessCount=11, AvailableMemoryMB=7268
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.log.dir so I do NOT create it in target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/82f324c2-90f0-aeef-8a58-7d02ed3b39ab/hadoop.tmp.dir so I do NOT create it in target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd, deleteOnExit=true
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/test.cache.data in system properties and HBase conf
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:44:51,073 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:44:51,073 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:44:51,074 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:44:51,086 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:44:51,264 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:44:51,474 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:44:51,477 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:44:51,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:44:51,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:44:51,479 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:44:51,479 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:44:51,479 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7305dd28{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:44:51,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dab95de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:44:51,570 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16ccf5f4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/java.io.tmpdir/jetty-localhost-44609-hadoop-hdfs-3_4_1-tests_jar-_-any-10854679748977692487/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:44:51,571 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@314e7370{HTTP/1.1, (http/1.1)}{localhost:44609}
2024-11-20T19:44:51,571 INFO  [Time-limited test {}] server.Server(415): Started @250645ms
2024-11-20T19:44:51,581 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:44:51,850 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:44:51,852 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:44:51,853 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:44:51,853 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:44:51,854 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:44:51,854 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7884e2a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:44:51,854 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31e425dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:44:51,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:51,911 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:51,947 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5739bf7a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/java.io.tmpdir/jetty-localhost-46419-hadoop-hdfs-3_4_1-tests_jar-_-any-16716532596540005486/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:44:51,948 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49059d65{HTTP/1.1, (http/1.1)}{localhost:46419}
2024-11-20T19:44:51,948 INFO  [Time-limited test {}] server.Server(415): Started @251022ms
2024-11-20T19:44:51,949 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:44:51,974 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:44:51,976 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:44:51,977 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:44:51,977 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:44:51,977 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:44:51,977 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52993998{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:44:51,977 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@216b0c63{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:44:52,072 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@470e396e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/java.io.tmpdir/jetty-localhost-38627-hadoop-hdfs-3_4_1-tests_jar-_-any-3186117089625605710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:44:52,073 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2f81fda7{HTTP/1.1, (http/1.1)}{localhost:38627}
2024-11-20T19:44:52,073 INFO  [Time-limited test {}] server.Server(415): Started @251147ms
2024-11-20T19:44:52,073 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:44:52,900 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:52,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:53,107 WARN  [Thread-1967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data1/current/BP-1785201491-172.17.0.3-1732131891097/current, will proceed with Du for space computation calculation, 
2024-11-20T19:44:53,107 WARN  [Thread-1968 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data2/current/BP-1785201491-172.17.0.3-1732131891097/current, will proceed with Du for space computation calculation, 
2024-11-20T19:44:53,126 WARN  [Thread-1931 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:44:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc633bed9c51a32a with lease ID 0x62bd41d2f5ee4e4b: Processing first storage report for DS-7c0871ba-1e35-4936-a872-6f7baee8745c from datanode DatanodeRegistration(127.0.0.1:36445, datanodeUuid=ffcb0e80-1eb2-4164-bf66-ea4f5acc814a, infoPort=45455, infoSecurePort=0, ipcPort=34337, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097)
2024-11-20T19:44:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc633bed9c51a32a with lease ID 0x62bd41d2f5ee4e4b: from storage DS-7c0871ba-1e35-4936-a872-6f7baee8745c node DatanodeRegistration(127.0.0.1:36445, datanodeUuid=ffcb0e80-1eb2-4164-bf66-ea4f5acc814a, infoPort=45455, infoSecurePort=0, ipcPort=34337, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:44:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc633bed9c51a32a with lease ID 0x62bd41d2f5ee4e4b: Processing first storage report for DS-5b2dfedf-5a47-446a-b722-05d55d4ed5d9 from datanode DatanodeRegistration(127.0.0.1:36445, datanodeUuid=ffcb0e80-1eb2-4164-bf66-ea4f5acc814a, infoPort=45455, infoSecurePort=0, ipcPort=34337, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097)
2024-11-20T19:44:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc633bed9c51a32a with lease ID 0x62bd41d2f5ee4e4b: from storage DS-5b2dfedf-5a47-446a-b722-05d55d4ed5d9 node DatanodeRegistration(127.0.0.1:36445, datanodeUuid=ffcb0e80-1eb2-4164-bf66-ea4f5acc814a, infoPort=45455, infoSecurePort=0, ipcPort=34337, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:44:53,235 WARN  [Thread-1979 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data4/current/BP-1785201491-172.17.0.3-1732131891097/current, will proceed with Du for space computation calculation, 
2024-11-20T19:44:53,235 WARN  [Thread-1978 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data3/current/BP-1785201491-172.17.0.3-1732131891097/current, will proceed with Du for space computation calculation, 
2024-11-20T19:44:53,257 WARN  [Thread-1954 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:44:53,260 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3477e60d661c2930 with lease ID 0x62bd41d2f5ee4e4c: Processing first storage report for DS-fe979219-3421-4327-842b-9d1c9a5a8491 from datanode DatanodeRegistration(127.0.0.1:45659, datanodeUuid=fc3258da-41f5-4fc6-bc91-fc1c027cc363, infoPort=42761, infoSecurePort=0, ipcPort=36867, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097)
2024-11-20T19:44:53,260 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3477e60d661c2930 with lease ID 0x62bd41d2f5ee4e4c: from storage DS-fe979219-3421-4327-842b-9d1c9a5a8491 node DatanodeRegistration(127.0.0.1:45659, datanodeUuid=fc3258da-41f5-4fc6-bc91-fc1c027cc363, infoPort=42761, infoSecurePort=0, ipcPort=36867, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:44:53,260 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3477e60d661c2930 with lease ID 0x62bd41d2f5ee4e4c: Processing first storage report for DS-762194c7-8988-46bf-b832-412aa030bafe from datanode DatanodeRegistration(127.0.0.1:45659, datanodeUuid=fc3258da-41f5-4fc6-bc91-fc1c027cc363, infoPort=42761, infoSecurePort=0, ipcPort=36867, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097)
2024-11-20T19:44:53,260 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3477e60d661c2930 with lease ID 0x62bd41d2f5ee4e4c: from storage DS-762194c7-8988-46bf-b832-412aa030bafe node DatanodeRegistration(127.0.0.1:45659, datanodeUuid=fc3258da-41f5-4fc6-bc91-fc1c027cc363, infoPort=42761, infoSecurePort=0, ipcPort=36867, storageInfo=lv=-57;cid=testClusterID;nsid=1105404472;c=1732131891097), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:44:53,304 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae
2024-11-20T19:44:53,307 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/zookeeper_0, clientPort=59573, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:44:53,308 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59573
2024-11-20T19:44:53,309 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,311 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,321 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:44:53,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:44:53,323 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34 with version=8
2024-11-20T19:44:53,323 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:44:53,325 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:44:53,326 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:44:53,327 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44543
2024-11-20T19:44:53,328 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44543 connecting to ZooKeeper ensemble=127.0.0.1:59573
2024-11-20T19:44:53,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445430x0, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:44:53,395 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44543-0x10159c72be50000 connected
2024-11-20T19:44:53,490 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,495 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:44:53,499 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34, hbase.cluster.distributed=false
2024-11-20T19:44:53,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:44:53,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44543
2024-11-20T19:44:53,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44543
2024-11-20T19:44:53,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44543
2024-11-20T19:44:53,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44543
2024-11-20T19:44:53,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44543
2024-11-20T19:44:53,515 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:44:53,516 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45817
2024-11-20T19:44:53,518 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45817 connecting to ZooKeeper ensemble=127.0.0.1:59573
2024-11-20T19:44:53,518 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,519 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458170x0, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:44:53,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45817-0x10159c72be50001 connected
2024-11-20T19:44:53,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:44:53,531 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:44:53,532 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:44:53,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:44:53,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:44:53,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45817
2024-11-20T19:44:53,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45817
2024-11-20T19:44:53,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45817
2024-11-20T19:44:53,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45817
2024-11-20T19:44:53,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45817
2024-11-20T19:44:53,547 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:44543
2024-11-20T19:44:53,547 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:44:53,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:44:53,558 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:44:53,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,568 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:44:53,569 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,44543,1732131893325 from backup master directory
2024-11-20T19:44:53,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:44:53,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:44:53,578 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:44:53,579 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,583 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/hbase.id] with ID: ebcb9240-8349-4728-94aa-25aa937836e8
2024-11-20T19:44:53,583 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/.tmp/hbase.id
2024-11-20T19:44:53,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:44:53,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:44:53,588 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/.tmp/hbase.id]:[hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/hbase.id]
2024-11-20T19:44:53,598 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:53,598 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:44:53,600 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T19:44:53,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:44:53,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:44:53,618 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:44:53,619 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:44:53,621 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:44:53,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:44:53,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:44:53,628 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store
2024-11-20T19:44:53,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:44:53,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:44:53,636 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:53,636 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:44:53,636 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131893636Disabling compacts and flushes for region at 1732131893636Disabling writes for close at 1732131893636Writing region close event to WAL at 1732131893636Closed at 1732131893636
2024-11-20T19:44:53,637 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/.initializing
2024-11-20T19:44:53,637 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/WALs/e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,639 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C44543%2C1732131893325, suffix=, logDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/WALs/e14ef07817f4,44543,1732131893325, archiveDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/oldWALs, maxLogs=10
2024-11-20T19:44:53,640 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C44543%2C1732131893325.1732131893640
2024-11-20T19:44:53,644 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/WALs/e14ef07817f4,44543,1732131893325/e14ef07817f4%2C44543%2C1732131893325.1732131893640
2024-11-20T19:44:53,645 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45455:45455),(127.0.0.1/127.0.0.1:42761:42761)]
2024-11-20T19:44:53,646 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:44:53,646 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:53,646 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,646 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,647 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:44:53,648 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:44:53,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,648 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:53,648 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:44:53,649 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:44:53,649 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,650 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:44:53,650 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:44:53,651 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:44:53,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,651 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:44:53,651 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:44:53,652 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:44:53,652 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,653 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:44:53,653 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,653 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,654 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,655 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,655 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,655 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:44:53,656 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:44:53,681 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:44:53,681 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852061, jitterRate=0.08345235884189606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:44:53,682 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131893646Initializing all the Stores at 1732131893647 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131893647Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131893647Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131893647Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131893647Cleaning up temporary data from old regions at 1732131893655 (+8 ms)Region opened successfully at 1732131893682 (+27 ms)
2024-11-20T19:44:53,682 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:44:53,684 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b28ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:44:53,685 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:44:53,685 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:44:53,685 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:44:53,685 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:44:53,686 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:44:53,686 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:44:53,686 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:44:53,688 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:44:53,689 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:44:53,699 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:44:53,699 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:44:53,700 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:44:53,710 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:44:53,710 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:44:53,711 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:44:53,720 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:44:53,721 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:44:53,731 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:44:53,733 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:44:53,741 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:44:53,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:44:53,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:44:53,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,752 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,44543,1732131893325, sessionid=0x10159c72be50000, setting cluster-up flag (Was=false)
2024-11-20T19:44:53,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,805 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:44:53,806 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:53,857 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:44:53,859 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,44543,1732131893325
2024-11-20T19:44:53,860 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:44:53,862 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:44:53,863 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:44:53,863 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:44:53,863 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,44543,1732131893325
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:44:53,865 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,866 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131923866
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:44:53,867 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:44:53,867 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:44:53,868 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:44:53,868 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:44:53,868 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:44:53,868 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131893868,5,FailOnTimeoutGroup]
2024-11-20T19:44:53,868 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131893868,5,FailOnTimeoutGroup]
2024-11-20T19:44:53,868 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,868 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:44:53,869 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,869 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,869 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,869 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:44:53,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:44:53,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:44:53,876 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:44:53,876 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34
2024-11-20T19:44:53,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:44:53,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:44:53,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:53,886 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:44:53,888 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:44:53,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,889 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:53,889 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:44:53,890 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:44:53,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,891 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:53,891 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:44:53,892 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:44:53,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,893 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:53,893 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:44:53,894 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:44:53,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:53,894 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:53,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:44:53,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740
2024-11-20T19:44:53,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740
2024-11-20T19:44:53,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:44:53,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:44:53,897 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:44:53,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:44:53,900 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:44:53,901 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834332, jitterRate=0.06090806424617767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:44:53,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131893885Initializing all the Stores at 1732131893886 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131893886Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131893886Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131893886Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131893886Cleaning up temporary data from old regions at 1732131893896 (+10 ms)Region opened successfully at 1732131893901 (+5 ms)
2024-11-20T19:44:53,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:44:53,902 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:44:53,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:44:53,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:44:53,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:44:53,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:53,902 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:44:53,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131893901Disabling compacts and flushes for region at 1732131893901Disabling writes for close at 1732131893902 (+1 ms)Writing region close event to WAL at 1732131893902Closed at 1732131893902
2024-11-20T19:44:53,903 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:44:53,903 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:44:53,903 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:44:53,905 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:44:53,906 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:44:53,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:53,936 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(746): ClusterId : ebcb9240-8349-4728-94aa-25aa937836e8
2024-11-20T19:44:53,936 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:44:53,948 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:44:53,948 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:44:53,958 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:44:53,959 DEBUG [RS:0;e14ef07817f4:45817 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3958f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:44:53,975 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:45817
2024-11-20T19:44:53,975 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:44:53,975 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:44:53,975 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:44:53,975 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,44543,1732131893325 with port=45817, startcode=1732131893515
2024-11-20T19:44:53,976 DEBUG [RS:0;e14ef07817f4:45817 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:44:53,977 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50991, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:44:53,978 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44543 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,45817,1732131893515
2024-11-20T19:44:53,978 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44543 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:53,979 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34
2024-11-20T19:44:53,979 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46377
2024-11-20T19:44:53,979 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:44:53,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:44:53,990 DEBUG [RS:0;e14ef07817f4:45817 {}] zookeeper.ZKUtil(111): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,45817,1732131893515
2024-11-20T19:44:53,990 WARN  [RS:0;e14ef07817f4:45817 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:44:53,990 INFO  [RS:0;e14ef07817f4:45817 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:44:53,990 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515
2024-11-20T19:44:53,990 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,45817,1732131893515]
2024-11-20T19:44:53,993 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:44:53,994 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:44:53,995 INFO  [RS:0;e14ef07817f4:45817 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:44:53,995 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,995 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:44:53,996 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:44:53,996 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:44:53,996 DEBUG [RS:0;e14ef07817f4:45817 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:53,997 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,45817,1732131893515-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:44:54,009 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:44:54,009 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,45817,1732131893515-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,009 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,010 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.Replication(171): e14ef07817f4,45817,1732131893515 started
2024-11-20T19:44:54,021 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,021 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,45817,1732131893515, RpcServer on e14ef07817f4/172.17.0.3:45817, sessionid=0x10159c72be50001
2024-11-20T19:44:54,021 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:44:54,021 DEBUG [RS:0;e14ef07817f4:45817 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,021 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,45817,1732131893515'
2024-11-20T19:44:54,021 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:44:54,022 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,45817,1732131893515'
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:44:54,023 DEBUG [RS:0;e14ef07817f4:45817 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:44:54,023 INFO  [RS:0;e14ef07817f4:45817 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:44:54,023 INFO  [RS:0;e14ef07817f4:45817 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:44:54,056 WARN  [e14ef07817f4:44543 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:44:54,128 INFO  [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C45817%2C1732131893515, suffix=, logDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515, archiveDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs, maxLogs=32
2024-11-20T19:44:54,130 INFO  [RS:0;e14ef07817f4:45817 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45817%2C1732131893515.1732131894129
2024-11-20T19:44:54,138 INFO  [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131894129
2024-11-20T19:44:54,144 DEBUG [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45455:45455),(127.0.0.1/127.0.0.1:42761:42761)]
2024-11-20T19:44:54,307 DEBUG [e14ef07817f4:44543 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:44:54,308 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,311 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,45817,1732131893515, state=OPENING
2024-11-20T19:44:54,368 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:44:54,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:54,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:44:54,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:44:54,380 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:44:54,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:44:54,380 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,45817,1732131893515}]
2024-11-20T19:44:54,535 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:44:54,539 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43661, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:44:54,546 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:44:54,546 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:44:54,550 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C45817%2C1732131893515.meta, suffix=.meta, logDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515, archiveDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs, maxLogs=32
2024-11-20T19:44:54,550 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45817%2C1732131893515.meta.1732131894550.meta
2024-11-20T19:44:54,556 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.meta.1732131894550.meta
2024-11-20T19:44:54,557 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45455:45455),(127.0.0.1/127.0.0.1:42761:42761)]
2024-11-20T19:44:54,558 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:44:54,558 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:44:54,558 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:44:54,559 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:44:54,559 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:44:54,559 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:54,559 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:44:54,559 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:44:54,560 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:44:54,561 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:44:54,561 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:54,562 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:54,562 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:44:54,562 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:44:54,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:54,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:54,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:44:54,564 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:44:54,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:54,564 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:54,564 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:44:54,565 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:44:54,565 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:54,565 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:44:54,565 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:44:54,566 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740
2024-11-20T19:44:54,567 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740
2024-11-20T19:44:54,569 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:44:54,569 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:44:54,569 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:44:54,571 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:44:54,572 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=839627, jitterRate=0.0676414966583252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:44:54,572 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:44:54,572 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131894559Writing region info on filesystem at 1732131894559Initializing all the Stores at 1732131894560 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131894560Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131894560Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131894560Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131894560Cleaning up temporary data from old regions at 1732131894569 (+9 ms)Running coprocessor post-open hooks at 1732131894572 (+3 ms)Region opened successfully at 1732131894572
2024-11-20T19:44:54,573 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131894535
2024-11-20T19:44:54,575 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:44:54,575 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:44:54,576 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,577 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,45817,1732131893515, state=OPEN
2024-11-20T19:44:54,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:44:54,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:44:54,611 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:44:54,611 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:44:54,616 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:44:54,616 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,45817,1732131893515 in 231 msec
2024-11-20T19:44:54,621 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:44:54,621 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 713 msec
2024-11-20T19:44:54,622 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:44:54,622 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:44:54,624 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:44:54,624 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,45817,1732131893515, seqNum=-1]
2024-11-20T19:44:54,624 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:44:54,626 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52753, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:44:54,634 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 770 msec
2024-11-20T19:44:54,634 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131894634, completionTime=-1
2024-11-20T19:44:54,634 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:44:54,634 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:44:54,637 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:44:54,637 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732131954637
2024-11-20T19:44:54,637 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732132014637
2024-11-20T19:44:54,637 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 3 msec
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:44543, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,638 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,642 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:44:54,644 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.065sec
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:44:54,645 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:44:54,648 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:44:54,648 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:44:54,648 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,44543,1732131893325-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:44:54,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a53521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:44:54,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,44543,-1 for getting cluster id
2024-11-20T19:44:54,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:44:54,740 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ebcb9240-8349-4728-94aa-25aa937836e8'
2024-11-20T19:44:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:44:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ebcb9240-8349-4728-94aa-25aa937836e8"

2024-11-20T19:44:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20021697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:44:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,44543,-1]
2024-11-20T19:44:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:44:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:44:54,745 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:44:54,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2a39d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:44:54,747 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:44:54,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,45817,1732131893515, seqNum=-1]
2024-11-20T19:44:54,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:44:54,750 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:44:54,752 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,44543,1732131893325
2024-11-20T19:44:54,753 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:44:54,756 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:44:54,756 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T19:44:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e14ef07817f4,44543,1732131893325
2024-11-20T19:44:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f98ea4a
2024-11-20T19:44:54,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T19:44:54,759 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T19:44:54,760 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T19:44:54,760 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T19:44:54,760 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:44:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-20T19:44:54,763 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T19:44:54,764 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:54,764 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-20T19:44:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:44:54,765 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T19:44:54,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741835_1011 (size=381)
2024-11-20T19:44:54,773 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741835_1011 (size=381)
2024-11-20T19:44:54,775 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d8e5a187420ec9abc9f28ab3a9f683d2, NAME => 'TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34
2024-11-20T19:44:54,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741836_1012 (size=64)
2024-11-20T19:44:54,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741836_1012 (size=64)
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d8e5a187420ec9abc9f28ab3a9f683d2, disabling compactions & flushes
2024-11-20T19:44:54,780 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. after waiting 0 ms
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:54,780 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:54,780 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d8e5a187420ec9abc9f28ab3a9f683d2:
Waiting for close lock at 1732131894780Disabling compacts and flushes for region at 1732131894780Disabling writes for close at 1732131894780Writing region close event to WAL at 1732131894780Closed at 1732131894780
2024-11-20T19:44:54,782 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T19:44:54,782 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732131894782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131894782"}]},"ts":"1732131894782"}
2024-11-20T19:44:54,784 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T19:44:54,785 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T19:44:54,786 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131894785"}]},"ts":"1732131894785"}
2024-11-20T19:44:54,788 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-20T19:44:54,788 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, ASSIGN}]
2024-11-20T19:44:54,789 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, ASSIGN
2024-11-20T19:44:54,790 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, ASSIGN; state=OFFLINE, location=e14ef07817f4,45817,1732131893515; forceNewPlan=false, retain=false
2024-11-20T19:44:54,902 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:54,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:54,942 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d8e5a187420ec9abc9f28ab3a9f683d2, regionState=OPENING, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:54,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, ASSIGN because future has completed
2024-11-20T19:44:54,948 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515}]
2024-11-20T19:44:55,113 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:55,113 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d8e5a187420ec9abc9f28ab3a9f683d2, NAME => 'TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:44:55,113 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,114 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:44:55,114 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,114 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,116 INFO  [StoreOpener-d8e5a187420ec9abc9f28ab3a9f683d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d8e5a187420ec9abc9f28ab3a9f683d2 
2024-11-20T19:44:55,119 INFO  [StoreOpener-d8e5a187420ec9abc9f28ab3a9f683d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8e5a187420ec9abc9f28ab3a9f683d2 columnFamilyName info
2024-11-20T19:44:55,119 DEBUG [StoreOpener-d8e5a187420ec9abc9f28ab3a9f683d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:44:55,120 INFO  [StoreOpener-d8e5a187420ec9abc9f28ab3a9f683d2-1 {}] regionserver.HStore(327): Store=d8e5a187420ec9abc9f28ab3a9f683d2/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:44:55,120 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,122 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,122 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,123 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,123 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,126 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,130 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:44:55,131 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d8e5a187420ec9abc9f28ab3a9f683d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800712, jitterRate=0.01815822720527649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:44:55,131 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:44:55,132 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d8e5a187420ec9abc9f28ab3a9f683d2:
Running coprocessor pre-open hook at 1732131895114Writing region info on filesystem at 1732131895114Initializing all the Stores at 1732131895116 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131895116Cleaning up temporary data from old regions at 1732131895123 (+7 ms)Running coprocessor post-open hooks at 1732131895131 (+8 ms)Region opened successfully at 1732131895132 (+1 ms)
2024-11-20T19:44:55,133 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., pid=6, masterSystemTime=1732131895104
2024-11-20T19:44:55,136 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:55,136 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:44:55,137 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d8e5a187420ec9abc9f28ab3a9f683d2, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:44:55,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515 because future has completed
2024-11-20T19:44:55,143 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T19:44:55,143 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515 in 193 msec
2024-11-20T19:44:55,146 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T19:44:55,146 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, ASSIGN in 355 msec
2024-11-20T19:44:55,147 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T19:44:55,147 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732131895147"}]},"ts":"1732131895147"}
2024-11-20T19:44:55,149 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-20T19:44:55,150 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T19:44:55,152 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 390 msec
2024-11-20T19:44:55,549 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,549 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,549 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,571 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,572 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,572 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,572 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,572 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,572 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,575 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,578 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:55,903 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:55,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:56,083 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:44:56,086 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,089 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,092 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,112 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,113 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,116 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,116 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,116 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,119 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:44:56,904 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:56,915 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:57,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-20T19:44:57,373 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T19:44:57,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T19:44:57,905 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:57,916 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:58,907 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:58,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:59,908 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:59,918 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:44:59,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:44:59,995 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-20T19:45:00,910 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:00,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:01,911 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:01,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:02,881 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:45:02,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,883 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,884 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,886 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,912 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,912 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:02,913 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,916 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,916 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:02,921 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:03,913 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:03,921 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:04,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T19:45:04,778 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-20T19:45:04,778 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-20T19:45:04,787 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-20T19:45:04,787 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2]
2024-11-20T19:45:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:04,804 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:04,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/ef1fd015fdb9450b88f542d907a17bc0 is 1080, key is row0001/info:/1732131904793/Put/seqid=0
2024-11-20T19:45:04,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741837_1013 (size=12509)
2024-11-20T19:45:04,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741837_1013 (size=12509)
2024-11-20T19:45:04,837 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/ef1fd015fdb9450b88f542d907a17bc0
2024-11-20T19:45:04,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/ef1fd015fdb9450b88f542d907a17bc0 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0
2024-11-20T19:45:04,850 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-20T19:45:04,851 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0, entries=7, sequenceid=11, filesize=12.2 K
2024-11-20T19:45:04,852 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for d8e5a187420ec9abc9f28ab3a9f683d2 in 49ms, sequenceid=11, compaction requested=false
2024-11-20T19:45:04,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:04,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:47910 deadline: 1732131914850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T19:45:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T19:45:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 because the exception is null or not the one we care about
2024-11-20T19:45:04,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:04,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:05,914 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:05,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:06,916 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:06,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:07,917 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:07,925 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:08,918 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:08,926 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:09,920 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:09,927 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:10,921 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:10,928 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:11,922 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:11,930 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:12,924 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:12,931 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:13,925 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:13,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:14,901 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-11-20T19:45:14,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/7275278149454ad5bc5e2bbeb351e12d is 1080, key is row0008/info:/1732131904805/Put/seqid=0
2024-11-20T19:45:14,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741838_1014 (size=29761)
2024-11-20T19:45:14,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741838_1014 (size=29761)
2024-11-20T19:45:14,916 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/7275278149454ad5bc5e2bbeb351e12d
2024-11-20T19:45:14,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/7275278149454ad5bc5e2bbeb351e12d as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d
2024-11-20T19:45:14,926 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:14,927 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d, entries=23, sequenceid=37, filesize=29.1 K
2024-11-20T19:45:14,928 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for d8e5a187420ec9abc9f28ab3a9f683d2 in 28ms, sequenceid=37, compaction requested=false
2024-11-20T19:45:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-11-20T19:45:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d because midkey is the same as first or last row
2024-11-20T19:45:14,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:15,927 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:15,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:16,927 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:16,928 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:16,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/991680a15e9148ceaf551fd095842188 is 1080, key is row0031/info:/1732131914904/Put/seqid=0
2024-11-20T19:45:16,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:16,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741839_1015 (size=12509)
2024-11-20T19:45:16,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741839_1015 (size=12509)
2024-11-20T19:45:16,940 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/991680a15e9148ceaf551fd095842188
2024-11-20T19:45:16,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/991680a15e9148ceaf551fd095842188 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188
2024-11-20T19:45:16,952 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188, entries=7, sequenceid=47, filesize=12.2 K
2024-11-20T19:45:16,953 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for d8e5a187420ec9abc9f28ab3a9f683d2 in 26ms, sequenceid=47, compaction requested=true
2024-11-20T19:45:16,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:16,953 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-11-20T19:45:16,953 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:16,953 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d because midkey is the same as first or last row
2024-11-20T19:45:16,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d8e5a187420ec9abc9f28ab3a9f683d2:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:16,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:16,954 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:16,954 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T19:45:16,955 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:16,955 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): d8e5a187420ec9abc9f28ab3a9f683d2/info is initiating minor compaction (all files)
2024-11-20T19:45:16,956 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d8e5a187420ec9abc9f28ab3a9f683d2/info in TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:16,956 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp, totalSize=53.5 K
2024-11-20T19:45:16,956 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1fd015fdb9450b88f542d907a17bc0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732131904793
2024-11-20T19:45:16,957 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7275278149454ad5bc5e2bbeb351e12d, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732131904805
2024-11-20T19:45:16,957 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991680a15e9148ceaf551fd095842188, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732131914904
2024-11-20T19:45:16,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/cc72110c28ef4297aaeb992e32b4ebcf is 1080, key is row0038/info:/1732131916929/Put/seqid=0
2024-11-20T19:45:16,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741840_1016 (size=17894)
2024-11-20T19:45:16,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741840_1016 (size=17894)
2024-11-20T19:45:16,967 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/cc72110c28ef4297aaeb992e32b4ebcf
2024-11-20T19:45:16,972 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d8e5a187420ec9abc9f28ab3a9f683d2#info#compaction#59 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:16,972 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/e83bc4d293b344738b19ac3e454a8eab is 1080, key is row0001/info:/1732131904793/Put/seqid=0
2024-11-20T19:45:16,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/cc72110c28ef4297aaeb992e32b4ebcf as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf
2024-11-20T19:45:16,978 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741841_1017 (size=44978)
2024-11-20T19:45:16,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741841_1017 (size=44978)
2024-11-20T19:45:16,979 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf, entries=12, sequenceid=62, filesize=17.5 K
2024-11-20T19:45:16,981 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=11.56 KB/11836 for d8e5a187420ec9abc9f28ab3a9f683d2 in 26ms, sequenceid=62, compaction requested=false
2024-11-20T19:45:16,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:16,981 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=71.0 K, sizeToCheck=16.0 K
2024-11-20T19:45:16,981 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:16,981 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d because midkey is the same as first or last row
2024-11-20T19:45:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:16,981 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T19:45:16,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/443f45319a5244e8a91d0bc78e3c9a93 is 1080, key is row0050/info:/1732131916954/Put/seqid=0
2024-11-20T19:45:16,987 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/e83bc4d293b344738b19ac3e454a8eab as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab
2024-11-20T19:45:16,993 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d8e5a187420ec9abc9f28ab3a9f683d2/info of d8e5a187420ec9abc9f28ab3a9f683d2 into e83bc4d293b344738b19ac3e454a8eab(size=43.9 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:16,993 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:16,993 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., storeName=d8e5a187420ec9abc9f28ab3a9f683d2/info, priority=13, startTime=1732131916953; duration=0sec
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab because midkey is the same as first or last row
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab because midkey is the same as first or last row
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=61.4 K, sizeToCheck=16.0 K
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab because midkey is the same as first or last row
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:16,994 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d8e5a187420ec9abc9f28ab3a9f683d2:info
2024-11-20T19:45:16,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741842_1018 (size=17894)
2024-11-20T19:45:16,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741842_1018 (size=17894)
2024-11-20T19:45:16,998 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/443f45319a5244e8a91d0bc78e3c9a93
2024-11-20T19:45:17,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/443f45319a5244e8a91d0bc78e3c9a93 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93
2024-11-20T19:45:17,008 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93, entries=12, sequenceid=77, filesize=17.5 K
2024-11-20T19:45:17,009 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=3.15 KB/3228 for d8e5a187420ec9abc9f28ab3a9f683d2 in 28ms, sequenceid=77, compaction requested=true
2024-11-20T19:45:17,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:17,009 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.9 K, sizeToCheck=16.0 K
2024-11-20T19:45:17,009 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:17,009 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab because midkey is the same as first or last row
2024-11-20T19:45:17,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d8e5a187420ec9abc9f28ab3a9f683d2:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:17,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:17,010 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:17,011 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 80766 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:17,011 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1541): d8e5a187420ec9abc9f28ab3a9f683d2/info is initiating minor compaction (all files)
2024-11-20T19:45:17,011 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d8e5a187420ec9abc9f28ab3a9f683d2/info in TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:17,011 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp, totalSize=78.9 K
2024-11-20T19:45:17,011 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting e83bc4d293b344738b19ac3e454a8eab, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732131904793
2024-11-20T19:45:17,011 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting cc72110c28ef4297aaeb992e32b4ebcf, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732131916929
2024-11-20T19:45:17,012 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting 443f45319a5244e8a91d0bc78e3c9a93, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732131916954
2024-11-20T19:45:17,024 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d8e5a187420ec9abc9f28ab3a9f683d2#info#compaction#61 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:17,024 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/01e0c1758ebd4bdaa3a0232707985f00 is 1080, key is row0001/info:/1732131904793/Put/seqid=0
2024-11-20T19:45:17,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741843_1019 (size=71001)
2024-11-20T19:45:17,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741843_1019 (size=71001)
2024-11-20T19:45:17,035 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/01e0c1758ebd4bdaa3a0232707985f00 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00
2024-11-20T19:45:17,041 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d8e5a187420ec9abc9f28ab3a9f683d2/info of d8e5a187420ec9abc9f28ab3a9f683d2 into 01e0c1758ebd4bdaa3a0232707985f00(size=69.3 K), total size for store is 69.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:17,041 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:17,041 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., storeName=d8e5a187420ec9abc9f28ab3a9f683d2/info, priority=13, startTime=1732131917009; duration=0sec
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:17,042 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d8e5a187420ec9abc9f28ab3a9f683d2:info
2024-11-20T19:45:17,929 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:17,935 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:18,930 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:18,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,000 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:19,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/e05f4e7a73ac43b7a44b524bc53c291f is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:19,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741844_1020 (size=12509)
2024-11-20T19:45:19,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741844_1020 (size=12509)
2024-11-20T19:45:19,014 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/e05f4e7a73ac43b7a44b524bc53c291f
2024-11-20T19:45:19,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/e05f4e7a73ac43b7a44b524bc53c291f as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f
2024-11-20T19:45:19,026 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f, entries=7, sequenceid=89, filesize=12.2 K
2024-11-20T19:45:19,027 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for d8e5a187420ec9abc9f28ab3a9f683d2 in 27ms, sequenceid=89, compaction requested=false
2024-11-20T19:45:19,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:19,027 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.6 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,028 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,028 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:19,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/deca59ca1ed54105a6d9350ca3239f59 is 1080, key is row0069/info:/1732131919002/Put/seqid=0
2024-11-20T19:45:19,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741845_1021 (size=16817)
2024-11-20T19:45:19,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741845_1021 (size=16817)
2024-11-20T19:45:19,038 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/deca59ca1ed54105a6d9350ca3239f59
2024-11-20T19:45:19,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/deca59ca1ed54105a6d9350ca3239f59 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59
2024-11-20T19:45:19,048 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59, entries=11, sequenceid=103, filesize=16.4 K
2024-11-20T19:45:19,049 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for d8e5a187420ec9abc9f28ab3a9f683d2 in 21ms, sequenceid=103, compaction requested=true
2024-11-20T19:45:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=98.0 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:19,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d8e5a187420ec9abc9f28ab3a9f683d2:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:19,050 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:19,051 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 100327 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:19,051 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): d8e5a187420ec9abc9f28ab3a9f683d2/info is initiating minor compaction (all files)
2024-11-20T19:45:19,051 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d8e5a187420ec9abc9f28ab3a9f683d2/info in TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,051 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp, totalSize=98.0 K
2024-11-20T19:45:19,051 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T19:45:19,051 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01e0c1758ebd4bdaa3a0232707985f00, keycount=61, bloomtype=ROW, size=69.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732131904793
2024-11-20T19:45:19,052 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05f4e7a73ac43b7a44b524bc53c291f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732131916983
2024-11-20T19:45:19,052 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting deca59ca1ed54105a6d9350ca3239f59, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732131919002
2024-11-20T19:45:19,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/629a0ba9eb004689972c316069f6f415 is 1080, key is row0080/info:/1732131919029/Put/seqid=0
2024-11-20T19:45:19,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741846_1022 (size=17894)
2024-11-20T19:45:19,060 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741846_1022 (size=17894)
2024-11-20T19:45:19,060 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/629a0ba9eb004689972c316069f6f415
2024-11-20T19:45:19,066 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d8e5a187420ec9abc9f28ab3a9f683d2#info#compaction#65 average throughput is 27.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:19,066 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/3a1817d6bd9945e08e7c7cb553234602 is 1080, key is row0001/info:/1732131904793/Put/seqid=0
2024-11-20T19:45:19,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/629a0ba9eb004689972c316069f6f415 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/629a0ba9eb004689972c316069f6f415
2024-11-20T19:45:19,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741847_1023 (size=90562)
2024-11-20T19:45:19,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741847_1023 (size=90562)
2024-11-20T19:45:19,071 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/629a0ba9eb004689972c316069f6f415, entries=12, sequenceid=118, filesize=17.5 K
2024-11-20T19:45:19,072 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for d8e5a187420ec9abc9f28ab3a9f683d2 in 21ms, sequenceid=118, compaction requested=false
2024-11-20T19:45:19,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:19,072 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=115.5 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,072 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,072 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 because midkey is the same as first or last row
2024-11-20T19:45:19,075 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/3a1817d6bd9945e08e7c7cb553234602 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602
2024-11-20T19:45:19,080 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d8e5a187420ec9abc9f28ab3a9f683d2/info of d8e5a187420ec9abc9f28ab3a9f683d2 into 3a1817d6bd9945e08e7c7cb553234602(size=88.4 K), total size for store is 105.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d8e5a187420ec9abc9f28ab3a9f683d2:

2024-11-20T19:45:19,080 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., storeName=d8e5a187420ec9abc9f28ab3a9f683d2/info, priority=13, startTime=1732131919049; duration=0sec
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T19:45:19,080 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T19:45:19,081 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:19,081 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:19,081 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d8e5a187420ec9abc9f28ab3a9f683d2:info
2024-11-20T19:45:19,082 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44543 {}] assignment.AssignmentManager(1355): Split request from e14ef07817f4,45817,1732131893515, parent={ENCODED => d8e5a187420ec9abc9f28ab3a9f683d2, NAME => 'TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-20T19:45:19,087 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44543 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:19,090 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44543 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d8e5a187420ec9abc9f28ab3a9f683d2, daughterA=084b4858c7fc9b851a3c3a1bf578271c, daughterB=25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:19,091 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d8e5a187420ec9abc9f28ab3a9f683d2, daughterA=084b4858c7fc9b851a3c3a1bf578271c, daughterB=25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:19,091 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d8e5a187420ec9abc9f28ab3a9f683d2, daughterA=084b4858c7fc9b851a3c3a1bf578271c, daughterB=25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:19,091 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d8e5a187420ec9abc9f28ab3a9f683d2, daughterA=084b4858c7fc9b851a3c3a1bf578271c, daughterB=25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:19,099 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, UNASSIGN}]
2024-11-20T19:45:19,100 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, UNASSIGN
2024-11-20T19:45:19,102 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d8e5a187420ec9abc9f28ab3a9f683d2, regionState=CLOSING, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:19,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, UNASSIGN because future has completed
2024-11-20T19:45:19,104 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-20T19:45:19,104 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515}]
2024-11-20T19:45:19,265 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,265 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-20T19:45:19,266 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing d8e5a187420ec9abc9f28ab3a9f683d2, disabling compactions & flushes
2024-11-20T19:45:19,267 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:19,267 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:19,267 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. after waiting 0 ms
2024-11-20T19:45:19,267 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:19,267 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing d8e5a187420ec9abc9f28ab3a9f683d2 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB
2024-11-20T19:45:19,272 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/89f360b3985c46acb1026f4d91ca2628 is 1080, key is row0092/info:/1732131919052/Put/seqid=0
2024-11-20T19:45:19,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741848_1024 (size=10347)
2024-11-20T19:45:19,277 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741848_1024 (size=10347)
2024-11-20T19:45:19,278 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/89f360b3985c46acb1026f4d91ca2628
2024-11-20T19:45:19,282 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/.tmp/info/89f360b3985c46acb1026f4d91ca2628 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/89f360b3985c46acb1026f4d91ca2628
2024-11-20T19:45:19,288 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/89f360b3985c46acb1026f4d91ca2628, entries=5, sequenceid=127, filesize=10.1 K
2024-11-20T19:45:19,289 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for d8e5a187420ec9abc9f28ab3a9f683d2 in 22ms, sequenceid=127, compaction requested=true
2024-11-20T19:45:19,290 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59] to archive
2024-11-20T19:45:19,291 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:45:19,293 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/ef1fd015fdb9450b88f542d907a17bc0
2024-11-20T19:45:19,294 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/7275278149454ad5bc5e2bbeb351e12d
2024-11-20T19:45:19,295 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e83bc4d293b344738b19ac3e454a8eab
2024-11-20T19:45:19,297 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/991680a15e9148ceaf551fd095842188
2024-11-20T19:45:19,298 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/cc72110c28ef4297aaeb992e32b4ebcf
2024-11-20T19:45:19,299 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/01e0c1758ebd4bdaa3a0232707985f00
2024-11-20T19:45:19,300 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/443f45319a5244e8a91d0bc78e3c9a93
2024-11-20T19:45:19,302 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/e05f4e7a73ac43b7a44b524bc53c291f
2024-11-20T19:45:19,303 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/deca59ca1ed54105a6d9350ca3239f59
2024-11-20T19:45:19,309 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-20T19:45:19,310 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.
2024-11-20T19:45:19,310 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for d8e5a187420ec9abc9f28ab3a9f683d2:
Waiting for close lock at 1732131919266Running coprocessor pre-close hooks at 1732131919266Disabling compacts and flushes for region at 1732131919266Disabling writes for close at 1732131919267 (+1 ms)Obtaining lock to block concurrent updates at 1732131919267Preparing flush snapshotting stores in d8e5a187420ec9abc9f28ab3a9f683d2 at 1732131919267Finished memstore snapshotting TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., syncing WAL and waiting on mvcc, flushsize=dataSize=5380, getHeapSize=6000, getOffHeapSize=0, getCellsCount=5 at 1732131919267Flushing stores of TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. at 1732131919268 (+1 ms)Flushing d8e5a187420ec9abc9f28ab3a9f683d2/info: creating writer at 1732131919268Flushing d8e5a187420ec9abc9f28ab3a9f683d2/info: appending metadata at 1732131919272 (+4 ms)Flushing d8e5a187420ec9abc9f28ab3a9f683d2/info: closing flushed file at 1732131919272Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56dcd3d8: reopening flushed file at 1732131919282 (+10 ms)Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for d8e5a187420ec9abc9f28ab3a9f683d2 in 22ms, sequenceid=127, compaction requested=true at 1732131919289 (+7 ms)Writing region close event to WAL at 1732131919305 (+16 ms)Running coprocessor post-close hooks at 1732131919310 (+5 ms)Closed at 1732131919310
2024-11-20T19:45:19,313 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,313 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d8e5a187420ec9abc9f28ab3a9f683d2, regionState=CLOSED
2024-11-20T19:45:19,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515 because future has completed
2024-11-20T19:45:19,318 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-20T19:45:19,318 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure d8e5a187420ec9abc9f28ab3a9f683d2, server=e14ef07817f4,45817,1732131893515 in 212 msec
2024-11-20T19:45:19,320 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-20T19:45:19,320 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d8e5a187420ec9abc9f28ab3a9f683d2, UNASSIGN in 219 msec
2024-11-20T19:45:19,327 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:19,330 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=d8e5a187420ec9abc9f28ab3a9f683d2, threads=3
2024-11-20T19:45:19,331 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,331 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/89f360b3985c46acb1026f4d91ca2628 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,331 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/629a0ba9eb004689972c316069f6f415 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,340 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/89f360b3985c46acb1026f4d91ca2628, top=true
2024-11-20T19:45:19,340 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/629a0ba9eb004689972c316069f6f415, top=true
2024-11-20T19:45:19,345 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628 for child: 25bb528bf2a59b6e7e6dba1ea4f1d5dc, parent: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,345 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/89f360b3985c46acb1026f4d91ca2628 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,349 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415 for child: 25bb528bf2a59b6e7e6dba1ea4f1d5dc, parent: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,349 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/629a0ba9eb004689972c316069f6f415 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741849_1025 (size=27)
2024-11-20T19:45:19,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741849_1025 (size=27)
2024-11-20T19:45:19,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741850_1026 (size=27)
2024-11-20T19:45:19,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741850_1026 (size=27)
2024-11-20T19:45:19,770 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602 for region: d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:19,772 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region d8e5a187420ec9abc9f28ab3a9f683d2 Daughter A: [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2] storefiles, Daughter B: [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628] storefiles.
2024-11-20T19:45:19,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741851_1027 (size=71)
2024-11-20T19:45:19,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741851_1027 (size=71)
2024-11-20T19:45:19,781 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:19,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741852_1028 (size=71)
2024-11-20T19:45:19,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741852_1028 (size=71)
2024-11-20T19:45:19,794 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:19,803 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-20T19:45:19,805 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-20T19:45:19,807 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732131919807"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732131919807"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732131919807"}]},"ts":"1732131919807"}
2024-11-20T19:45:19,807 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732131919807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131919807"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732131919807"}]},"ts":"1732131919807"}
2024-11-20T19:45:19,807 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732131919807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732131919807"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732131919807"}]},"ts":"1732131919807"}
2024-11-20T19:45:19,823 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=084b4858c7fc9b851a3c3a1bf578271c, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=25bb528bf2a59b6e7e6dba1ea4f1d5dc, ASSIGN}]
2024-11-20T19:45:19,824 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=084b4858c7fc9b851a3c3a1bf578271c, ASSIGN
2024-11-20T19:45:19,824 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=25bb528bf2a59b6e7e6dba1ea4f1d5dc, ASSIGN
2024-11-20T19:45:19,825 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=084b4858c7fc9b851a3c3a1bf578271c, ASSIGN; state=SPLITTING_NEW, location=e14ef07817f4,45817,1732131893515; forceNewPlan=false, retain=false
2024-11-20T19:45:19,825 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=25bb528bf2a59b6e7e6dba1ea4f1d5dc, ASSIGN; state=SPLITTING_NEW, location=e14ef07817f4,45817,1732131893515; forceNewPlan=false, retain=false
2024-11-20T19:45:19,931 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:19,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:19,976 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=084b4858c7fc9b851a3c3a1bf578271c, regionState=OPENING, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:19,976 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=25bb528bf2a59b6e7e6dba1ea4f1d5dc, regionState=OPENING, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:19,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=25bb528bf2a59b6e7e6dba1ea4f1d5dc, ASSIGN because future has completed
2024-11-20T19:45:19,979 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25bb528bf2a59b6e7e6dba1ea4f1d5dc, server=e14ef07817f4,45817,1732131893515}]
2024-11-20T19:45:19,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=084b4858c7fc9b851a3c3a1bf578271c, ASSIGN because future has completed
2024-11-20T19:45:19,980 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 084b4858c7fc9b851a3c3a1bf578271c, server=e14ef07817f4,45817,1732131893515}]
2024-11-20T19:45:20,137 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:20,138 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 084b4858c7fc9b851a3c3a1bf578271c, NAME => 'TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-20T19:45:20,138 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,138 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:20,138 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,138 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,141 INFO  [StoreOpener-084b4858c7fc9b851a3c3a1bf578271c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 084b4858c7fc9b851a3c3a1bf578271c 
2024-11-20T19:45:20,143 INFO  [StoreOpener-084b4858c7fc9b851a3c3a1bf578271c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 084b4858c7fc9b851a3c3a1bf578271c columnFamilyName info
2024-11-20T19:45:20,143 DEBUG [StoreOpener-084b4858c7fc9b851a3c3a1bf578271c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:20,157 DEBUG [StoreOpener-084b4858c7fc9b851a3c3a1bf578271c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-bottom
2024-11-20T19:45:20,158 INFO  [StoreOpener-084b4858c7fc9b851a3c3a1bf578271c-1 {}] regionserver.HStore(327): Store=084b4858c7fc9b851a3c3a1bf578271c/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:45:20,158 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,159 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,160 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,160 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,160 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,162 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,163 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 084b4858c7fc9b851a3c3a1bf578271c; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860783, jitterRate=0.09454265236854553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:45:20,163 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:20,163 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 084b4858c7fc9b851a3c3a1bf578271c:
Running coprocessor pre-open hook at 1732131920139Writing region info on filesystem at 1732131920139Initializing all the Stores at 1732131920140 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131920140Cleaning up temporary data from old regions at 1732131920160 (+20 ms)Running coprocessor post-open hooks at 1732131920163 (+3 ms)Region opened successfully at 1732131920163
2024-11-20T19:45:20,164 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c., pid=13, masterSystemTime=1732131920131
2024-11-20T19:45:20,164 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 084b4858c7fc9b851a3c3a1bf578271c:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:20,164 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-20T19:45:20,164 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:20,165 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:20,165 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 084b4858c7fc9b851a3c3a1bf578271c/info is initiating minor compaction (all files)
2024-11-20T19:45:20,165 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 084b4858c7fc9b851a3c3a1bf578271c/info in TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:20,165 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-bottom] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/.tmp, totalSize=88.4 K
2024-11-20T19:45:20,166 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732131904793
2024-11-20T19:45:20,166 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:20,167 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:20,167 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:20,167 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 25bb528bf2a59b6e7e6dba1ea4f1d5dc, NAME => 'TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-20T19:45:20,167 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,167 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:20,167 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=084b4858c7fc9b851a3c3a1bf578271c, regionState=OPEN, openSeqNum=131, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:20,167 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,167 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,168 INFO  [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 25bb528bf2a59b6e7e6dba1ea4f1d5dc 
2024-11-20T19:45:20,169 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-20T19:45:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-20T19:45:20,169 INFO  [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25bb528bf2a59b6e7e6dba1ea4f1d5dc columnFamilyName info
2024-11-20T19:45:20,169 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-20T19:45:20,169 DEBUG [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:20,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 084b4858c7fc9b851a3c3a1bf578271c, server=e14ef07817f4,45817,1732131893515 because future has completed
2024-11-20T19:45:20,173 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-20T19:45:20,173 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 084b4858c7fc9b851a3c3a1bf578271c, server=e14ef07817f4,45817,1732131893515 in 191 msec
2024-11-20T19:45:20,174 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=084b4858c7fc9b851a3c3a1bf578271c, ASSIGN in 350 msec
2024-11-20T19:45:20,183 DEBUG [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-top
2024-11-20T19:45:20,185 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 084b4858c7fc9b851a3c3a1bf578271c#info#compaction#67 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:20,185 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/.tmp/info/25b2c37d53254220a5246447df5c60da is 1080, key is row0001/info:/1732131904793/Put/seqid=0
2024-11-20T19:45:20,188 DEBUG [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415
2024-11-20T19:45:20,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/55e1cb2ae3714697acfb2b7e843c68e2 is 193, key is TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc./info:regioninfo/1732131919976/Put/seqid=0
2024-11-20T19:45:20,193 DEBUG [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628
2024-11-20T19:45:20,193 INFO  [StoreOpener-25bb528bf2a59b6e7e6dba1ea4f1d5dc-1 {}] regionserver.HStore(327): Store=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:45:20,193 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741853_1029 (size=70862)
2024-11-20T19:45:20,196 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741853_1029 (size=70862)
2024-11-20T19:45:20,198 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,198 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,198 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,200 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,201 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 25bb528bf2a59b6e7e6dba1ea4f1d5dc; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=778170, jitterRate=-0.010506346821784973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T19:45:20,201 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:20,201 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:
Running coprocessor pre-open hook at 1732131920167Writing region info on filesystem at 1732131920167Initializing all the Stores at 1732131920168 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131920168Cleaning up temporary data from old regions at 1732131920198 (+30 ms)Running coprocessor post-open hooks at 1732131920201 (+3 ms)Region opened successfully at 1732131920201
2024-11-20T19:45:20,202 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., pid=12, masterSystemTime=1732131920131
2024-11-20T19:45:20,202 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 2
2024-11-20T19:45:20,202 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:20,202 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:20,203 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/.tmp/info/25b2c37d53254220a5246447df5c60da as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/25b2c37d53254220a5246447df5c60da
2024-11-20T19:45:20,203 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:20,203 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:20,203 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:20,203 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-top, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=116.0 K
2024-11-20T19:45:20,204 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732131904793
2024-11-20T19:45:20,204 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732131919029
2024-11-20T19:45:20,204 DEBUG [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:20,204 INFO  [RS_OPEN_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:20,204 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732131919052
2024-11-20T19:45:20,205 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=25bb528bf2a59b6e7e6dba1ea4f1d5dc, regionState=OPEN, openSeqNum=131, regionLocation=e14ef07817f4,45817,1732131893515
2024-11-20T19:45:20,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741854_1030 (size=9847)
2024-11-20T19:45:20,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741854_1030 (size=9847)
2024-11-20T19:45:20,206 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/55e1cb2ae3714697acfb2b7e843c68e2
2024-11-20T19:45:20,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25bb528bf2a59b6e7e6dba1ea4f1d5dc, server=e14ef07817f4,45817,1732131893515 because future has completed
2024-11-20T19:45:20,210 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 084b4858c7fc9b851a3c3a1bf578271c/info of 084b4858c7fc9b851a3c3a1bf578271c into 25b2c37d53254220a5246447df5c60da(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:20,210 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 084b4858c7fc9b851a3c3a1bf578271c:

2024-11-20T19:45:20,210 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c., storeName=084b4858c7fc9b851a3c3a1bf578271c/info, priority=15, startTime=1732131920164; duration=0sec
2024-11-20T19:45:20,210 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:20,210 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 084b4858c7fc9b851a3c3a1bf578271c:info
2024-11-20T19:45:20,213 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-20T19:45:20,213 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 25bb528bf2a59b6e7e6dba1ea4f1d5dc, server=e14ef07817f4,45817,1732131893515 in 231 msec
2024-11-20T19:45:20,215 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-20T19:45:20,215 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=25bb528bf2a59b6e7e6dba1ea4f1d5dc, ASSIGN in 390 msec
2024-11-20T19:45:20,217 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d8e5a187420ec9abc9f28ab3a9f683d2, daughterA=084b4858c7fc9b851a3c3a1bf578271c, daughterB=25bb528bf2a59b6e7e6dba1ea4f1d5dc in 1.1280 sec
2024-11-20T19:45:20,227 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#69 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:20,227 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/d032d37ac7df46b3afff574fa7e70556 is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:20,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/ns/58d8aed3661e4d618557feeef1d1ba06 is 43, key is default/ns:d/1732131894627/Put/seqid=0
2024-11-20T19:45:20,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741855_1031 (size=42984)
2024-11-20T19:45:20,235 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741855_1031 (size=42984)
2024-11-20T19:45:20,241 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/d032d37ac7df46b3afff574fa7e70556 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d032d37ac7df46b3afff574fa7e70556
2024-11-20T19:45:20,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741856_1032 (size=5153)
2024-11-20T19:45:20,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741856_1032 (size=5153)
2024-11-20T19:45:20,242 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/ns/58d8aed3661e4d618557feeef1d1ba06
2024-11-20T19:45:20,248 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into d032d37ac7df46b3afff574fa7e70556(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:20,248 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:20,248 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131920202; duration=0sec
2024-11-20T19:45:20,248 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:20,248 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:20,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/table/7b1d41a509e64b49be2268ec529f8a7f is 65, key is TestLogRolling-testLogRolling/table:state/1732131895147/Put/seqid=0
2024-11-20T19:45:20,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741857_1033 (size=5340)
2024-11-20T19:45:20,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741857_1033 (size=5340)
2024-11-20T19:45:20,266 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/table/7b1d41a509e64b49be2268ec529f8a7f
2024-11-20T19:45:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/55e1cb2ae3714697acfb2b7e843c68e2 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/info/55e1cb2ae3714697acfb2b7e843c68e2
2024-11-20T19:45:20,277 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/info/55e1cb2ae3714697acfb2b7e843c68e2, entries=30, sequenceid=17, filesize=9.6 K
2024-11-20T19:45:20,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/ns/58d8aed3661e4d618557feeef1d1ba06 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/ns/58d8aed3661e4d618557feeef1d1ba06
2024-11-20T19:45:20,283 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/ns/58d8aed3661e4d618557feeef1d1ba06, entries=2, sequenceid=17, filesize=5.0 K
2024-11-20T19:45:20,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/table/7b1d41a509e64b49be2268ec529f8a7f as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/table/7b1d41a509e64b49be2268ec529f8a7f
2024-11-20T19:45:20,289 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/table/7b1d41a509e64b49be2268ec529f8a7f, entries=2, sequenceid=17, filesize=5.2 K
2024-11-20T19:45:20,290 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 120ms, sequenceid=17, compaction requested=false
2024-11-20T19:45:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-20T19:45:20,932 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:20,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:47910 deadline: 1732131931065, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. is not online on e14ef07817f4,45817,1732131893515
2024-11-20T19:45:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. is not online on e14ef07817f4,45817,1732131893515
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T19:45:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2. is not online on e14ef07817f4,45817,1732131893515
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T19:45:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732131894760.d8e5a187420ec9abc9f28ab3a9f683d2., hostname=e14ef07817f4,45817,1732131893515, seqNum=2 from cache
2024-11-20T19:45:21,933 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:21,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:22,934 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:22,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:23,304 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T19:45:23,935 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:23,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,311 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,852 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T19:45:24,855 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,855 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,856 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,856 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,857 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,858 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,881 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,881 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,881 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,882 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,882 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,882 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,885 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T19:45:24,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:24,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:25,936 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:25,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:26,937 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:26,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:27,938 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:27,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:28,939 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:28,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:29,940 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:29,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:30,941 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:30,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., hostname=e14ef07817f4,45817,1732131893515, seqNum=131]
2024-11-20T19:45:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:31,192 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:31,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/417f583dc792490699ba3cd65a36e513 is 1080, key is row0097/info:/1732131931180/Put/seqid=0
2024-11-20T19:45:31,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741858_1034 (size=12516)
2024-11-20T19:45:31,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741858_1034 (size=12516)
2024-11-20T19:45:31,201 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/417f583dc792490699ba3cd65a36e513
2024-11-20T19:45:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/417f583dc792490699ba3cd65a36e513 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513
2024-11-20T19:45:31,213 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513, entries=7, sequenceid=141, filesize=12.2 K
2024-11-20T19:45:31,214 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 22ms, sequenceid=141, compaction requested=false
2024-11-20T19:45:31,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:31,215 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T19:45:31,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/15bc51fc94394de5b2ee22e03d482a70 is 1080, key is row0104/info:/1732131931193/Put/seqid=0
2024-11-20T19:45:31,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741859_1035 (size=17906)
2024-11-20T19:45:31,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741859_1035 (size=17906)
2024-11-20T19:45:31,224 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/15bc51fc94394de5b2ee22e03d482a70
2024-11-20T19:45:31,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/15bc51fc94394de5b2ee22e03d482a70 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70
2024-11-20T19:45:31,236 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70, entries=12, sequenceid=156, filesize=17.5 K
2024-11-20T19:45:31,236 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 21ms, sequenceid=156, compaction requested=true
2024-11-20T19:45:31,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:31,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:31,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:31,237 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:31,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:31,238 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:31,238 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 73406 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:31,238 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:31,238 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:31,238 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d032d37ac7df46b3afff574fa7e70556, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=71.7 K
2024-11-20T19:45:31,239 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting d032d37ac7df46b3afff574fa7e70556, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732131916983
2024-11-20T19:45:31,239 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting 417f583dc792490699ba3cd65a36e513, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732131931180
2024-11-20T19:45:31,239 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] compactions.Compactor(225): Compacting 15bc51fc94394de5b2ee22e03d482a70, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732131931193
2024-11-20T19:45:31,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0981df544e514b4bb779bd704c65d8b9 is 1080, key is row0116/info:/1732131931216/Put/seqid=0
2024-11-20T19:45:31,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741860_1036 (size=16828)
2024-11-20T19:45:31,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741860_1036 (size=16828)
2024-11-20T19:45:31,250 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#75 average throughput is 27.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:31,251 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/936652e14ece4a199eb200496de534ad is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:31,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741861_1037 (size=63636)
2024-11-20T19:45:31,256 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741861_1037 (size=63636)
2024-11-20T19:45:31,261 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/936652e14ece4a199eb200496de534ad as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/936652e14ece4a199eb200496de534ad
2024-11-20T19:45:31,266 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into 936652e14ece4a199eb200496de534ad(size=62.1 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:31,266 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:31,266 INFO  [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131931237; duration=0sec
2024-11-20T19:45:31,266 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:31,266 DEBUG [RS:0;e14ef07817f4:45817-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:31,647 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0981df544e514b4bb779bd704c65d8b9
2024-11-20T19:45:31,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0981df544e514b4bb779bd704c65d8b9 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9
2024-11-20T19:45:31,664 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9, entries=11, sequenceid=170, filesize=16.4 K
2024-11-20T19:45:31,666 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=2.10 KB/2152 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 429ms, sequenceid=170, compaction requested=false
2024-11-20T19:45:31,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:31,942 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:31,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:32,943 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:32,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:33,263 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:33,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/eb21146629f3453ea434af41b537d5d9 is 1080, key is row0127/info:/1732131931239/Put/seqid=0
2024-11-20T19:45:33,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741862_1038 (size=12516)
2024-11-20T19:45:33,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741862_1038 (size=12516)
2024-11-20T19:45:33,276 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/eb21146629f3453ea434af41b537d5d9
2024-11-20T19:45:33,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/eb21146629f3453ea434af41b537d5d9 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9
2024-11-20T19:45:33,288 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9, entries=7, sequenceid=181, filesize=12.2 K
2024-11-20T19:45:33,289 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 26ms, sequenceid=181, compaction requested=true
2024-11-20T19:45:33,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:33,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:33,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:33,290 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:33,290 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T19:45:33,291 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 92980 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:33,291 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:33,291 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:33,291 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/936652e14ece4a199eb200496de534ad, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=90.8 K
2024-11-20T19:45:33,292 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936652e14ece4a199eb200496de534ad, keycount=54, bloomtype=ROW, size=62.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732131916983
2024-11-20T19:45:33,292 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0981df544e514b4bb779bd704c65d8b9, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732131931216
2024-11-20T19:45:33,292 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb21146629f3453ea434af41b537d5d9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732131931239
2024-11-20T19:45:33,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/b5f9460c998c473ebc28fda31793060e is 1080, key is row0134/info:/1732131933265/Put/seqid=0
2024-11-20T19:45:33,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741863_1039 (size=17906)
2024-11-20T19:45:33,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741863_1039 (size=17906)
2024-11-20T19:45:33,308 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/b5f9460c998c473ebc28fda31793060e
2024-11-20T19:45:33,309 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#78 average throughput is 36.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:33,309 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/64527aa67d154fa7bad41c0337ed958d is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:33,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/b5f9460c998c473ebc28fda31793060e as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e
2024-11-20T19:45:33,318 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e, entries=12, sequenceid=196, filesize=17.5 K
2024-11-20T19:45:33,319 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=14.71 KB/15064 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 29ms, sequenceid=196, compaction requested=false
2024-11-20T19:45:33,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:33,320 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-20T19:45:33,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741864_1040 (size=83215)
2024-11-20T19:45:33,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741864_1040 (size=83215)
2024-11-20T19:45:33,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/32eaa74f76f945249110af9af72323c8 is 1080, key is row0146/info:/1732131933291/Put/seqid=0
2024-11-20T19:45:33,327 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/64527aa67d154fa7bad41c0337ed958d as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/64527aa67d154fa7bad41c0337ed958d
2024-11-20T19:45:33,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741865_1041 (size=21156)
2024-11-20T19:45:33,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741865_1041 (size=21156)
2024-11-20T19:45:33,328 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/32eaa74f76f945249110af9af72323c8
2024-11-20T19:45:33,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/32eaa74f76f945249110af9af72323c8 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8
2024-11-20T19:45:33,333 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into 64527aa67d154fa7bad41c0337ed958d(size=81.3 K), total size for store is 98.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:33,333 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:33,333 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131933290; duration=0sec
2024-11-20T19:45:33,334 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:33,334 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:33,337 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8, entries=15, sequenceid=214, filesize=20.7 K
2024-11-20T19:45:33,338 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 18ms, sequenceid=214, compaction requested=true
2024-11-20T19:45:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:33,338 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:33,339 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 122277 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:33,339 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:33,339 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:33,339 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/64527aa67d154fa7bad41c0337ed958d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=119.4 K
2024-11-20T19:45:33,340 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64527aa67d154fa7bad41c0337ed958d, keycount=72, bloomtype=ROW, size=81.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732131916983
2024-11-20T19:45:33,340 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5f9460c998c473ebc28fda31793060e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732131933265
2024-11-20T19:45:33,341 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32eaa74f76f945249110af9af72323c8, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732131933291
2024-11-20T19:45:33,351 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#80 average throughput is 50.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:33,352 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/50945816750945b5aebe851707171639 is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:33,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741866_1042 (size=112431)
2024-11-20T19:45:33,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741866_1042 (size=112431)
2024-11-20T19:45:33,361 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/50945816750945b5aebe851707171639 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/50945816750945b5aebe851707171639
2024-11-20T19:45:33,366 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into 50945816750945b5aebe851707171639(size=109.8 K), total size for store is 109.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:33,367 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:33,367 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131933338; duration=0sec
2024-11-20T19:45:33,367 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:33,367 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:33,944 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:33,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:34,662 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T19:45:34,662 INFO  [master/e14ef07817f4:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T19:45:34,945 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:34,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:35,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:35,343 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:35,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ca3f23cb505f43ccbd1f0277eed714c0 is 1080, key is row0161/info:/1732131935323/Put/seqid=0
2024-11-20T19:45:35,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741867_1043 (size=12516)
2024-11-20T19:45:35,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741867_1043 (size=12516)
2024-11-20T19:45:35,353 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ca3f23cb505f43ccbd1f0277eed714c0
2024-11-20T19:45:35,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ca3f23cb505f43ccbd1f0277eed714c0 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0
2024-11-20T19:45:35,366 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0, entries=7, sequenceid=226, filesize=12.2 K
2024-11-20T19:45:35,367 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 24ms, sequenceid=226, compaction requested=false
2024-11-20T19:45:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:35,367 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:35,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0072d1d01b014cc29bf7727106271b1b is 1080, key is row0168/info:/1732131935344/Put/seqid=0
2024-11-20T19:45:35,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741868_1044 (size=16828)
2024-11-20T19:45:35,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741868_1044 (size=16828)
2024-11-20T19:45:35,376 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0072d1d01b014cc29bf7727106271b1b
2024-11-20T19:45:35,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0072d1d01b014cc29bf7727106271b1b as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b
2024-11-20T19:45:35,386 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b, entries=11, sequenceid=240, filesize=16.4 K
2024-11-20T19:45:35,387 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 20ms, sequenceid=240, compaction requested=true
2024-11-20T19:45:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:35,387 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:35,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:35,388 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:35,388 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 141775 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:35,388 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:35,388 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:35,389 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/50945816750945b5aebe851707171639, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=138.5 K
2024-11-20T19:45:35,389 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50945816750945b5aebe851707171639, keycount=99, bloomtype=ROW, size=109.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732131916983
2024-11-20T19:45:35,389 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca3f23cb505f43ccbd1f0277eed714c0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732131935323
2024-11-20T19:45:35,390 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0072d1d01b014cc29bf7727106271b1b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732131935344
2024-11-20T19:45:35,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ad5330c70fd54c5aa79648061bf5e927 is 1080, key is row0179/info:/1732131935368/Put/seqid=0
2024-11-20T19:45:35,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741869_1045 (size=16828)
2024-11-20T19:45:35,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741869_1045 (size=16828)
2024-11-20T19:45:35,397 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ad5330c70fd54c5aa79648061bf5e927
2024-11-20T19:45:35,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/ad5330c70fd54c5aa79648061bf5e927 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927
2024-11-20T19:45:35,403 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#84 average throughput is 40.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:35,403 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/6b91b8a0bc6944a2bd4b0b7f42d712b6 is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:35,406 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927, entries=11, sequenceid=254, filesize=16.4 K
2024-11-20T19:45:35,407 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=3.15 KB/3228 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 19ms, sequenceid=254, compaction requested=false
2024-11-20T19:45:35,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:35,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741870_1046 (size=132069)
2024-11-20T19:45:35,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741870_1046 (size=132069)
2024-11-20T19:45:35,415 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/6b91b8a0bc6944a2bd4b0b7f42d712b6 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/6b91b8a0bc6944a2bd4b0b7f42d712b6
2024-11-20T19:45:35,420 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into 6b91b8a0bc6944a2bd4b0b7f42d712b6(size=129.0 K), total size for store is 145.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:35,420 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:35,420 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131935387; duration=0sec
2024-11-20T19:45:35,420 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:35,420 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:35,946 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:35,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:36,947 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:36,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:37,407 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:37,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/fc32ffdb461b4e23b514a3d40257b9b0 is 1080, key is row0190/info:/1732131935389/Put/seqid=0
2024-11-20T19:45:37,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741871_1047 (size=12523)
2024-11-20T19:45:37,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741871_1047 (size=12523)
2024-11-20T19:45:37,417 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/fc32ffdb461b4e23b514a3d40257b9b0
2024-11-20T19:45:37,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/fc32ffdb461b4e23b514a3d40257b9b0 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0
2024-11-20T19:45:37,428 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0, entries=7, sequenceid=265, filesize=12.2 K
2024-11-20T19:45:37,429 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 22ms, sequenceid=265, compaction requested=true
2024-11-20T19:45:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:37,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:37,429 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:37,431 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 161420 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:37,431 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:37,431 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:37,431 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/6b91b8a0bc6944a2bd4b0b7f42d712b6, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=157.6 K
2024-11-20T19:45:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:37,431 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:37,431 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b91b8a0bc6944a2bd4b0b7f42d712b6, keycount=117, bloomtype=ROW, size=129.0 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732131916983
2024-11-20T19:45:37,432 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad5330c70fd54c5aa79648061bf5e927, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732131935368
2024-11-20T19:45:37,432 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc32ffdb461b4e23b514a3d40257b9b0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732131935389
2024-11-20T19:45:37,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/5028cfa9449d49bdba16db8edc5abd3b is 1080, key is row0197/info:/1732131937408/Put/seqid=0
2024-11-20T19:45:37,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741872_1048 (size=16839)
2024-11-20T19:45:37,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741872_1048 (size=16839)
2024-11-20T19:45:37,448 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/5028cfa9449d49bdba16db8edc5abd3b
2024-11-20T19:45:37,449 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#87 average throughput is 46.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:37,451 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/5a59c7fb2f824c79a2a96b89f2a1a0dd is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:37,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/5028cfa9449d49bdba16db8edc5abd3b as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b
2024-11-20T19:45:37,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741873_1049 (size=151655)
2024-11-20T19:45:37,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741873_1049 (size=151655)
2024-11-20T19:45:37,462 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b, entries=11, sequenceid=279, filesize=16.4 K
2024-11-20T19:45:37,463 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=14.71 KB/15064 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 31ms, sequenceid=279, compaction requested=false
2024-11-20T19:45:37,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:37,463 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-20T19:45:37,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/8822be7552cd45738af22046f9f30ec5 is 1080, key is row0208/info:/1732131937432/Put/seqid=0
2024-11-20T19:45:37,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741874_1050 (size=21171)
2024-11-20T19:45:37,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741874_1050 (size=21171)
2024-11-20T19:45:37,477 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/8822be7552cd45738af22046f9f30ec5
2024-11-20T19:45:37,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/8822be7552cd45738af22046f9f30ec5 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5
2024-11-20T19:45:37,486 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5, entries=15, sequenceid=297, filesize=20.7 K
2024-11-20T19:45:37,487 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=2.10 KB/2152 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 24ms, sequenceid=297, compaction requested=false
2024-11-20T19:45:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:37,872 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/5a59c7fb2f824c79a2a96b89f2a1a0dd as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5a59c7fb2f824c79a2a96b89f2a1a0dd
2024-11-20T19:45:37,880 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into 5a59c7fb2f824c79a2a96b89f2a1a0dd(size=148.1 K), total size for store is 185.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:37,880 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:37,880 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131937429; duration=0sec
2024-11-20T19:45:37,880 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:37,880 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:37,948 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:37,952 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:38,949 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:38,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:39,483 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T19:45:39,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/880718b336e74ed799ac0dc19e0d8e2d is 1080, key is row0223/info:/1732131937464/Put/seqid=0
2024-11-20T19:45:39,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741875_1051 (size=12523)
2024-11-20T19:45:39,495 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741875_1051 (size=12523)
2024-11-20T19:45:39,496 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/880718b336e74ed799ac0dc19e0d8e2d
2024-11-20T19:45:39,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/880718b336e74ed799ac0dc19e0d8e2d as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d
2024-11-20T19:45:39,507 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d, entries=7, sequenceid=308, filesize=12.2 K
2024-11-20T19:45:39,508 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 25ms, sequenceid=308, compaction requested=true
2024-11-20T19:45:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:39,509 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-20T19:45:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:39,510 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T19:45:39,510 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4  files of size 202188 starting at candidate #0 after considering 3 permutations with 3 in ratio
2024-11-20T19:45:39,510 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:39,510 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:39,511 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5a59c7fb2f824c79a2a96b89f2a1a0dd, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=197.4 K
2024-11-20T19:45:39,511 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a59c7fb2f824c79a2a96b89f2a1a0dd, keycount=135, bloomtype=ROW, size=148.1 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732131916983
2024-11-20T19:45:39,511 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5028cfa9449d49bdba16db8edc5abd3b, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732131937408
2024-11-20T19:45:39,512 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8822be7552cd45738af22046f9f30ec5, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732131937432
2024-11-20T19:45:39,512 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880718b336e74ed799ac0dc19e0d8e2d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732131937464
2024-11-20T19:45:39,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0dc1ff823ae343ba887668517ea8802d is 1080, key is row0230/info:/1732131939484/Put/seqid=0
2024-11-20T19:45:39,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741876_1052 (size=16839)
2024-11-20T19:45:39,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741876_1052 (size=16839)
2024-11-20T19:45:39,520 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0dc1ff823ae343ba887668517ea8802d
2024-11-20T19:45:39,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0dc1ff823ae343ba887668517ea8802d as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d
2024-11-20T19:45:39,532 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#91 average throughput is 57.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:39,533 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/eea262ba12c04d43a639d3e72e6e5f7d is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:39,536 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d, entries=11, sequenceid=322, filesize=16.4 K
2024-11-20T19:45:39,536 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=12.61 KB/12912 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 26ms, sequenceid=322, compaction requested=false
2024-11-20T19:45:39,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45817 {}] regionserver.HRegion(8855): Flush requested on 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:39,538 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-20T19:45:39,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741877_1053 (size=187410)
2024-11-20T19:45:39,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741877_1053 (size=187410)
2024-11-20T19:45:39,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/2b3e34ec336b408fa6366c4a875a0285 is 1080, key is row0241/info:/1732131939511/Put/seqid=0
2024-11-20T19:45:39,548 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/eea262ba12c04d43a639d3e72e6e5f7d as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eea262ba12c04d43a639d3e72e6e5f7d
2024-11-20T19:45:39,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741878_1054 (size=19013)
2024-11-20T19:45:39,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741878_1054 (size=19013)
2024-11-20T19:45:39,550 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/2b3e34ec336b408fa6366c4a875a0285
2024-11-20T19:45:39,554 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into eea262ba12c04d43a639d3e72e6e5f7d(size=183.0 K), total size for store is 199.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:39,554 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:39,554 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=12, startTime=1732131939509; duration=0sec
2024-11-20T19:45:39,554 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:39,554 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:39,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/2b3e34ec336b408fa6366c4a875a0285 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285
2024-11-20T19:45:39,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-11-20T19:45:39,560 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285, entries=13, sequenceid=338, filesize=18.6 K
2024-11-20T19:45:39,561 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=3.15 KB/3228 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 23ms, sequenceid=338, compaction requested=true
2024-11-20T19:45:39,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:39,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T19:45:39,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:39,561 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T19:45:39,562 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 223262 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T19:45:39,562 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1541): 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info is initiating minor compaction (all files)
2024-11-20T19:45:39,562 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info in TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:39,562 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eea262ba12c04d43a639d3e72e6e5f7d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285] into tmpdir=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp, totalSize=218.0 K
2024-11-20T19:45:39,563 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea262ba12c04d43a639d3e72e6e5f7d, keycount=168, bloomtype=ROW, size=183.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732131916983
2024-11-20T19:45:39,563 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc1ff823ae343ba887668517ea8802d, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732131939484
2024-11-20T19:45:39,563 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3e34ec336b408fa6366c4a875a0285, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732131939511
2024-11-20T19:45:39,574 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25bb528bf2a59b6e7e6dba1ea4f1d5dc#info#compaction#93 average throughput is 65.67 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T19:45:39,574 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/d6c67a587ed44f9b99f29b27f2844fef is 1080, key is row0062/info:/1732131916983/Put/seqid=0
2024-11-20T19:45:39,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741879_1055 (size=213481)
2024-11-20T19:45:39,577 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741879_1055 (size=213481)
2024-11-20T19:45:39,581 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/d6c67a587ed44f9b99f29b27f2844fef as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d6c67a587ed44f9b99f29b27f2844fef
2024-11-20T19:45:39,587 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25bb528bf2a59b6e7e6dba1ea4f1d5dc/info of 25bb528bf2a59b6e7e6dba1ea4f1d5dc into d6c67a587ed44f9b99f29b27f2844fef(size=208.5 K), total size for store is 208.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T19:45:39,587 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:39,587 INFO  [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., storeName=25bb528bf2a59b6e7e6dba1ea4f1d5dc/info, priority=13, startTime=1732131939561; duration=0sec
2024-11-20T19:45:39,587 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T19:45:39,587 DEBUG [RS:0;e14ef07817f4:45817-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25bb528bf2a59b6e7e6dba1ea4f1d5dc:info
2024-11-20T19:45:39,950 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:39,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:40,951 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:40,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:41,545 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-20T19:45:41,546 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45817%2C1732131893515.1732131941546
2024-11-20T19:45:41,559 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,559 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,559 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,559 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,560 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,560 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131894129 with entries=320, filesize=311.04 KB; new WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131941546
2024-11-20T19:45:41,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741833_1009 (size=318509)
2024-11-20T19:45:41,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741833_1009 (size=318509)
2024-11-20T19:45:41,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42761:42761),(127.0.0.1/127.0.0.1:45455:45455)]
2024-11-20T19:45:41,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131894129 is not closed yet, will try archiving it next time
2024-11-20T19:45:41,566 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 084b4858c7fc9b851a3c3a1bf578271c:

2024-11-20T19:45:41,566 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 25bb528bf2a59b6e7e6dba1ea4f1d5dc 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-20T19:45:41,571 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0f4b732769b64f95823ffdbb6dc5d11b is 1080, key is row0254/info:/1732131939539/Put/seqid=0
2024-11-20T19:45:41,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741881_1057 (size=8199)
2024-11-20T19:45:41,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741881_1057 (size=8199)
2024-11-20T19:45:41,578 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0f4b732769b64f95823ffdbb6dc5d11b
2024-11-20T19:45:41,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/.tmp/info/0f4b732769b64f95823ffdbb6dc5d11b as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0f4b732769b64f95823ffdbb6dc5d11b
2024-11-20T19:45:41,587 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0f4b732769b64f95823ffdbb6dc5d11b, entries=3, sequenceid=346, filesize=8.0 K
2024-11-20T19:45:41,588 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 25bb528bf2a59b6e7e6dba1ea4f1d5dc in 22ms, sequenceid=346, compaction requested=false
2024-11-20T19:45:41,589 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:

2024-11-20T19:45:41,589 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-20T19:45:41,593 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/3835eca554ee40bcb5136a9a097f75f3 is 193, key is TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc./info:regioninfo/1732131920205/Put/seqid=0
2024-11-20T19:45:41,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741882_1058 (size=6223)
2024-11-20T19:45:41,597 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741882_1058 (size=6223)
2024-11-20T19:45:41,597 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/3835eca554ee40bcb5136a9a097f75f3
2024-11-20T19:45:41,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/.tmp/info/3835eca554ee40bcb5136a9a097f75f3 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/info/3835eca554ee40bcb5136a9a097f75f3
2024-11-20T19:45:41,606 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/info/3835eca554ee40bcb5136a9a097f75f3, entries=5, sequenceid=21, filesize=6.1 K
2024-11-20T19:45:41,607 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-11-20T19:45:41,607 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-20T19:45:41,608 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C45817%2C1732131893515.1732131941608
2024-11-20T19:45:41,613 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,613 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,613 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,613 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,613 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,613 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131941546 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131941608
2024-11-20T19:45:41,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741880_1056 (size=731)
2024-11-20T19:45:41,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741880_1056 (size=731)
2024-11-20T19:45:41,616 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131894129 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs/e14ef07817f4%2C45817%2C1732131893515.1732131894129
2024-11-20T19:45:41,617 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/WALs/e14ef07817f4,45817,1732131893515/e14ef07817f4%2C45817%2C1732131893515.1732131941546 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs/e14ef07817f4%2C45817%2C1732131893515.1732131941546
2024-11-20T19:45:41,622 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42761:42761),(127.0.0.1/127.0.0.1:45455:45455)]
2024-11-20T19:45:41,623 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-20T19:45:41,623 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:45:41,623 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:45:41,623 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:41,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:41,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:41,623 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:45:41,623 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:45:41,623 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=908490465, stopped=false
2024-11-20T19:45:41,623 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,44543,1732131893325
2024-11-20T19:45:41,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:41,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:41,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:41,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:41,680 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:45:41,680 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:45:41,681 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:41,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:41,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:41,681 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,45817,1732131893515' *****
2024-11-20T19:45:41,681 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:45:41,681 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:45:41,681 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(3091): Received CLOSE for 084b4858c7fc9b851a3c3a1bf578271c
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(3091): Received CLOSE for 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:41,681 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,45817,1732131893515
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 084b4858c7fc9b851a3c3a1bf578271c, disabling compactions & flushes
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:45817.
2024-11-20T19:45:41,682 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:41,682 DEBUG [RS:0;e14ef07817f4:45817 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c. after waiting 0 ms
2024-11-20T19:45:41,682 DEBUG [RS:0;e14ef07817f4:45817 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:45:41,682 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-20T19:45:41,682 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1325): Online Regions={084b4858c7fc9b851a3c3a1bf578271c=TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c., 25bb528bf2a59b6e7e6dba1ea4f1d5dc=TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:45:41,682 DEBUG [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1351): Waiting on 084b4858c7fc9b851a3c3a1bf578271c, 1588230740, 25bb528bf2a59b6e7e6dba1ea4f1d5dc
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:45:41,682 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:45:41,682 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-bottom] to archive
2024-11-20T19:45:41,682 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:45:41,683 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:45:41,685 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:41,685 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e14ef07817f4:44543 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-20T19:45:41,685 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-20T19:45:41,687 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-20T19:45:41,687 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:45:41,688 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:45:41,688 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131941682Running coprocessor pre-close hooks at 1732131941682Disabling compacts and flushes for region at 1732131941682Disabling writes for close at 1732131941682Writing region close event to WAL at 1732131941683 (+1 ms)Running coprocessor post-close hooks at 1732131941687 (+4 ms)Closed at 1732131941688 (+1 ms)
2024-11-20T19:45:41,688 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/084b4858c7fc9b851a3c3a1bf578271c/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-20T19:45:41,689 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 084b4858c7fc9b851a3c3a1bf578271c:
Waiting for close lock at 1732131941681Running coprocessor pre-close hooks at 1732131941681Disabling compacts and flushes for region at 1732131941681Disabling writes for close at 1732131941682 (+1 ms)Writing region close event to WAL at 1732131941686 (+4 ms)Running coprocessor post-close hooks at 1732131941689 (+3 ms)Closed at 1732131941689
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732131919087.084b4858c7fc9b851a3c3a1bf578271c.
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25bb528bf2a59b6e7e6dba1ea4f1d5dc, disabling compactions & flushes
2024-11-20T19:45:41,689 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc. after waiting 0 ms
2024-11-20T19:45:41,689 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:41,690 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2->hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/d8e5a187420ec9abc9f28ab3a9f683d2/info/3a1817d6bd9945e08e7c7cb553234602-top, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d032d37ac7df46b3afff574fa7e70556, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/936652e14ece4a199eb200496de534ad, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/64527aa67d154fa7bad41c0337ed958d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/50945816750945b5aebe851707171639, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/6b91b8a0bc6944a2bd4b0b7f42d712b6, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5a59c7fb2f824c79a2a96b89f2a1a0dd, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eea262ba12c04d43a639d3e72e6e5f7d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285] to archive
2024-11-20T19:45:41,691 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T19:45:41,692 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/3a1817d6bd9945e08e7c7cb553234602.d8e5a187420ec9abc9f28ab3a9f683d2
2024-11-20T19:45:41,693 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-629a0ba9eb004689972c316069f6f415
2024-11-20T19:45:41,695 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d032d37ac7df46b3afff574fa7e70556 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/d032d37ac7df46b3afff574fa7e70556
2024-11-20T19:45:41,696 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/TestLogRolling-testLogRolling=d8e5a187420ec9abc9f28ab3a9f683d2-89f360b3985c46acb1026f4d91ca2628
2024-11-20T19:45:41,697 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/417f583dc792490699ba3cd65a36e513
2024-11-20T19:45:41,698 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/936652e14ece4a199eb200496de534ad to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/936652e14ece4a199eb200496de534ad
2024-11-20T19:45:41,699 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/15bc51fc94394de5b2ee22e03d482a70
2024-11-20T19:45:41,700 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0981df544e514b4bb779bd704c65d8b9
2024-11-20T19:45:41,701 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/64527aa67d154fa7bad41c0337ed958d to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/64527aa67d154fa7bad41c0337ed958d
2024-11-20T19:45:41,702 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eb21146629f3453ea434af41b537d5d9
2024-11-20T19:45:41,703 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/b5f9460c998c473ebc28fda31793060e
2024-11-20T19:45:41,704 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/50945816750945b5aebe851707171639 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/50945816750945b5aebe851707171639
2024-11-20T19:45:41,704 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/32eaa74f76f945249110af9af72323c8
2024-11-20T19:45:41,705 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ca3f23cb505f43ccbd1f0277eed714c0
2024-11-20T19:45:41,706 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/6b91b8a0bc6944a2bd4b0b7f42d712b6 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/6b91b8a0bc6944a2bd4b0b7f42d712b6
2024-11-20T19:45:41,707 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0072d1d01b014cc29bf7727106271b1b
2024-11-20T19:45:41,708 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/ad5330c70fd54c5aa79648061bf5e927
2024-11-20T19:45:41,709 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5a59c7fb2f824c79a2a96b89f2a1a0dd to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5a59c7fb2f824c79a2a96b89f2a1a0dd
2024-11-20T19:45:41,710 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/fc32ffdb461b4e23b514a3d40257b9b0
2024-11-20T19:45:41,711 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/5028cfa9449d49bdba16db8edc5abd3b
2024-11-20T19:45:41,711 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/8822be7552cd45738af22046f9f30ec5
2024-11-20T19:45:41,712 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eea262ba12c04d43a639d3e72e6e5f7d to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/eea262ba12c04d43a639d3e72e6e5f7d
2024-11-20T19:45:41,713 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/880718b336e74ed799ac0dc19e0d8e2d
2024-11-20T19:45:41,714 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/0dc1ff823ae343ba887668517ea8802d
2024-11-20T19:45:41,715 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285 to hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/archive/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/info/2b3e34ec336b408fa6366c4a875a0285
2024-11-20T19:45:41,715 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d032d37ac7df46b3afff574fa7e70556=42984, 417f583dc792490699ba3cd65a36e513=12516, 936652e14ece4a199eb200496de534ad=63636, 15bc51fc94394de5b2ee22e03d482a70=17906, 0981df544e514b4bb779bd704c65d8b9=16828, 64527aa67d154fa7bad41c0337ed958d=83215, eb21146629f3453ea434af41b537d5d9=12516, b5f9460c998c473ebc28fda31793060e=17906, 50945816750945b5aebe851707171639=112431, 32eaa74f76f945249110af9af72323c8=21156, ca3f23cb505f43ccbd1f0277eed714c0=12516, 6b91b8a0bc6944a2bd4b0b7f42d712b6=132069, 0072d1d01b014cc29bf7727106271b1b=16828, ad5330c70fd54c5aa79648061bf5e927=16828, 5a59c7fb2f824c79a2a96b89f2a1a0dd=151655, fc32ffdb461b4e23b514a3d40257b9b0=12523, 5028cfa9449d49bdba16db8edc5abd3b=16839, 8822be7552cd45738af22046f9f30ec5=21171, eea262ba12c04d43a639d3e72e6e5f7d=187410, 880718b336e74ed799ac0dc19e0d8e2d=12523, 0dc1ff823ae343ba887668517ea8802d=16839, 2b3e34ec336b408fa6366c4a875a0285=19013]
2024-11-20T19:45:41,718 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/data/default/TestLogRolling-testLogRolling/25bb528bf2a59b6e7e6dba1ea4f1d5dc/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=130
2024-11-20T19:45:41,719 INFO  [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:41,719 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25bb528bf2a59b6e7e6dba1ea4f1d5dc:
Waiting for close lock at 1732131941689Running coprocessor pre-close hooks at 1732131941689Disabling compacts and flushes for region at 1732131941689Disabling writes for close at 1732131941689Writing region close event to WAL at 1732131941716 (+27 ms)Running coprocessor post-close hooks at 1732131941719 (+3 ms)Closed at 1732131941719
2024-11-20T19:45:41,719 DEBUG [RS_CLOSE_REGION-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732131919087.25bb528bf2a59b6e7e6dba1ea4f1d5dc.
2024-11-20T19:45:41,882 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,45817,1732131893515; all regions closed.
2024-11-20T19:45:41,883 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,883 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,883 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,883 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,883 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741834_1010 (size=8107)
2024-11-20T19:45:41,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741834_1010 (size=8107)
2024-11-20T19:45:41,887 DEBUG [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs
2024-11-20T19:45:41,887 INFO  [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C45817%2C1732131893515.meta:.meta(num 1732131894550)
2024-11-20T19:45:41,887 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,887 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,887 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,887 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,887 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:41,888 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741883_1059 (size=780)
2024-11-20T19:45:41,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741883_1059 (size=780)
2024-11-20T19:45:41,891 DEBUG [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/oldWALs
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C45817%2C1732131893515:(num 1732131941608)
2024-11-20T19:45:41,891 DEBUG [RS:0;e14ef07817f4:45817 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:45:41,891 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:45:41,891 INFO  [RS:0;e14ef07817f4:45817 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45817
2024-11-20T19:45:41,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,45817,1732131893515
2024-11-20T19:45:41,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:45:41,901 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:45:41,912 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,45817,1732131893515]
2024-11-20T19:45:41,922 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,45817,1732131893515 already deleted, retry=false
2024-11-20T19:45:41,922 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,45817,1732131893515 expired; onlineServers=0
2024-11-20T19:45:41,922 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,44543,1732131893325' *****
2024-11-20T19:45:41,922 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:45:41,923 INFO  [M:0;e14ef07817f4:44543 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:45:41,923 INFO  [M:0;e14ef07817f4:44543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:45:41,923 DEBUG [M:0;e14ef07817f4:44543 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:45:41,923 DEBUG [M:0;e14ef07817f4:44543 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:45:41,923 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:45:41,923 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131893868 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131893868,5,FailOnTimeoutGroup]
2024-11-20T19:45:41,923 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131893868 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131893868,5,FailOnTimeoutGroup]
2024-11-20T19:45:41,924 INFO  [M:0;e14ef07817f4:44543 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:45:41,924 INFO  [M:0;e14ef07817f4:44543 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:45:41,924 DEBUG [M:0;e14ef07817f4:44543 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:45:41,924 INFO  [M:0;e14ef07817f4:44543 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:45:41,924 INFO  [M:0;e14ef07817f4:44543 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:45:41,925 INFO  [M:0;e14ef07817f4:44543 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:45:41,925 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:45:41,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:45:41,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:41,933 DEBUG [M:0;e14ef07817f4:44543 {}] zookeeper.ZKUtil(347): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:45:41,933 WARN  [M:0;e14ef07817f4:44543 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:45:41,934 INFO  [M:0;e14ef07817f4:44543 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/.lastflushedseqids
2024-11-20T19:45:41,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741884_1060 (size=228)
2024-11-20T19:45:41,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741884_1060 (size=228)
2024-11-20T19:45:41,946 INFO  [M:0;e14ef07817f4:44543 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:45:41,946 INFO  [M:0;e14ef07817f4:44543 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:45:41,946 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:45:41,947 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:41,947 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:41,947 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:45:41,947 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:41,947 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-11-20T19:45:41,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:41,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:41,964 DEBUG [M:0;e14ef07817f4:44543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/476e69c9433740ecb8ebb5f61b23b9dd is 82, key is hbase:meta,,1/info:regioninfo/1732131894576/Put/seqid=0
2024-11-20T19:45:41,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741885_1061 (size=5672)
2024-11-20T19:45:41,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741885_1061 (size=5672)
2024-11-20T19:45:41,969 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/476e69c9433740ecb8ebb5f61b23b9dd
2024-11-20T19:45:41,984 DEBUG [M:0;e14ef07817f4:44543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbab0e8bdc8b41079ecdbb4bdf2d3297 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732131895151/Put/seqid=0
2024-11-20T19:45:41,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741886_1062 (size=7090)
2024-11-20T19:45:41,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741886_1062 (size=7090)
2024-11-20T19:45:41,990 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbab0e8bdc8b41079ecdbb4bdf2d3297
2024-11-20T19:45:41,993 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbab0e8bdc8b41079ecdbb4bdf2d3297
2024-11-20T19:45:42,000 INFO  [regionserver/e14ef07817f4:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:45:42,006 DEBUG [M:0;e14ef07817f4:44543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c38f3d17f944ea28898311a3ccd964a is 69, key is e14ef07817f4,45817,1732131893515/rs:state/1732131893978/Put/seqid=0
2024-11-20T19:45:42,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741887_1063 (size=5156)
2024-11-20T19:45:42,010 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741887_1063 (size=5156)
2024-11-20T19:45:42,011 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c38f3d17f944ea28898311a3ccd964a
2024-11-20T19:45:42,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:42,012 INFO  [RS:0;e14ef07817f4:45817 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:45:42,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45817-0x10159c72be50001, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:42,012 INFO  [RS:0;e14ef07817f4:45817 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,45817,1732131893515; zookeeper connection closed.
2024-11-20T19:45:42,012 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6da87b70 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6da87b70
2024-11-20T19:45:42,012 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:45:42,033 DEBUG [M:0;e14ef07817f4:44543 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e406aa8d928249c4b4268b9cdde74ee2 is 52, key is load_balancer_on/state:d/1732131894755/Put/seqid=0
2024-11-20T19:45:42,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741888_1064 (size=5056)
2024-11-20T19:45:42,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741888_1064 (size=5056)
2024-11-20T19:45:42,037 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e406aa8d928249c4b4268b9cdde74ee2
2024-11-20T19:45:42,042 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/476e69c9433740ecb8ebb5f61b23b9dd as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/476e69c9433740ecb8ebb5f61b23b9dd
2024-11-20T19:45:42,047 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/476e69c9433740ecb8ebb5f61b23b9dd, entries=8, sequenceid=125, filesize=5.5 K
2024-11-20T19:45:42,048 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbab0e8bdc8b41079ecdbb4bdf2d3297 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbab0e8bdc8b41079ecdbb4bdf2d3297
2024-11-20T19:45:42,053 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dbab0e8bdc8b41079ecdbb4bdf2d3297
2024-11-20T19:45:42,053 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbab0e8bdc8b41079ecdbb4bdf2d3297, entries=13, sequenceid=125, filesize=6.9 K
2024-11-20T19:45:42,054 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7c38f3d17f944ea28898311a3ccd964a as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c38f3d17f944ea28898311a3ccd964a
2024-11-20T19:45:42,059 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7c38f3d17f944ea28898311a3ccd964a, entries=1, sequenceid=125, filesize=5.0 K
2024-11-20T19:45:42,060 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e406aa8d928249c4b4268b9cdde74ee2 as hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e406aa8d928249c4b4268b9cdde74ee2
2024-11-20T19:45:42,065 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46377/user/jenkins/test-data/231d5ca0-892c-b94f-a004-2bace4cf2c34/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e406aa8d928249c4b4268b9cdde74ee2, entries=1, sequenceid=125, filesize=4.9 K
2024-11-20T19:45:42,066 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=125, compaction requested=false
2024-11-20T19:45:42,067 INFO  [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:42,068 DEBUG [M:0;e14ef07817f4:44543 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131941946Disabling compacts and flushes for region at 1732131941946Disabling writes for close at 1732131941947 (+1 ms)Obtaining lock to block concurrent updates at 1732131941947Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131941947Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1732131941947Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131941948 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131941948Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131941964 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131941964Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131941972 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131941984 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131941984Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131941993 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131942005 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131942005Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131942015 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131942032 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131942032Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@551d7c09: reopening flushed file at 1732131942041 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@211a3501: reopening flushed file at 1732131942047 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65cd8a01: reopening flushed file at 1732131942053 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62bcca2b: reopening flushed file at 1732131942059 (+6 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=125, compaction requested=false at 1732131942066 (+7 ms)Writing region close event to WAL at 1732131942067 (+1 ms)Closed at 1732131942067
2024-11-20T19:45:42,068 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:42,068 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:42,068 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:42,068 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:42,068 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:42,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45659 is added to blk_1073741830_1006 (size=61332)
2024-11-20T19:45:42,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36445 is added to blk_1073741830_1006 (size=61332)
2024-11-20T19:45:42,070 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:45:42,070 INFO  [M:0;e14ef07817f4:44543 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:45:42,070 INFO  [M:0;e14ef07817f4:44543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44543
2024-11-20T19:45:42,071 INFO  [M:0;e14ef07817f4:44543 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:45:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:42,208 INFO  [M:0;e14ef07817f4:44543 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:45:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44543-0x10159c72be50000, quorum=127.0.0.1:59573, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:42,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@470e396e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:42,213 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2f81fda7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:42,213 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:42,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@216b0c63{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:42,213 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52993998{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:42,216 WARN  [BP-1785201491-172.17.0.3-1732131891097 heartbeating to localhost/127.0.0.1:46377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:45:42,216 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:45:42,216 WARN  [BP-1785201491-172.17.0.3-1732131891097 heartbeating to localhost/127.0.0.1:46377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1785201491-172.17.0.3-1732131891097 (Datanode Uuid fc3258da-41f5-4fc6-bc91-fc1c027cc363) service to localhost/127.0.0.1:46377
2024-11-20T19:45:42,216 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:45:42,217 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data3/current/BP-1785201491-172.17.0.3-1732131891097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:42,217 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data4/current/BP-1785201491-172.17.0.3-1732131891097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:42,218 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:45:42,219 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5739bf7a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:42,220 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49059d65{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:42,220 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:42,220 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31e425dc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:42,220 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7884e2a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:42,221 WARN  [BP-1785201491-172.17.0.3-1732131891097 heartbeating to localhost/127.0.0.1:46377 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:45:42,221 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:45:42,221 WARN  [BP-1785201491-172.17.0.3-1732131891097 heartbeating to localhost/127.0.0.1:46377 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1785201491-172.17.0.3-1732131891097 (Datanode Uuid ffcb0e80-1eb2-4164-bf66-ea4f5acc814a) service to localhost/127.0.0.1:46377
2024-11-20T19:45:42,221 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:45:42,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data1/current/BP-1785201491-172.17.0.3-1732131891097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:42,222 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/cluster_a39731b2-07a3-480d-0f8a-83e5697db3dd/data/data2/current/BP-1785201491-172.17.0.3-1732131891097 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:42,222 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:45:42,228 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16ccf5f4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:45:42,228 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@314e7370{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:42,228 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:42,228 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dab95de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:42,228 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7305dd28{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:42,236 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:45:42,264 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:45:42,270 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 206)
Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46377
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46377
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46377 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46377 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46377 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46377
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46377
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:46377
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=57 (was 65), ProcessCount=11 (was 11), AvailableMemoryMB=7162 (was 7268)
2024-11-20T19:45:42,277 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=57, ProcessCount=11, AvailableMemoryMB=7162
2024-11-20T19:45:42,277 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T19:45:42,277 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.log.dir so I do NOT create it in target/test-data/a725a847-9894-3548-08e0-5344f571ac52
2024-11-20T19:45:42,277 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/71f0f1cf-67b6-b352-3eac-a4e29b7affae/hadoop.tmp.dir so I do NOT create it in target/test-data/a725a847-9894-3548-08e0-5344f571ac52
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1, deleteOnExit=true
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/test.cache.data in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T19:45:42,278 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T19:45:42,278 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/nfs.dump.dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/java.io.tmpdir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T19:45:42,279 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T19:45:42,307 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:45:42,739 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:45:42,742 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:45:42,743 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:45:42,743 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:45:42,743 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T19:45:42,744 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:45:42,744 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3910812a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:45:42,744 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a2ae1b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:45:42,837 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a7b167c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/java.io.tmpdir/jetty-localhost-38799-hadoop-hdfs-3_4_1-tests_jar-_-any-18265063907303076096/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:45:42,837 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@735ef7ff{HTTP/1.1, (http/1.1)}{localhost:38799}
2024-11-20T19:45:42,837 INFO  [Time-limited test {}] server.Server(415): Started @301912ms
2024-11-20T19:45:42,847 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T19:45:42,953 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:42,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:43,105 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:45:43,107 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:45:43,108 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:45:43,108 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:45:43,108 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:45:43,108 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1f1185ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:45:43,108 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ec9da7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:45:43,199 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4bb82bb7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/java.io.tmpdir/jetty-localhost-42057-hadoop-hdfs-3_4_1-tests_jar-_-any-5642469945217624260/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:43,200 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6a35d82b{HTTP/1.1, (http/1.1)}{localhost:42057}
2024-11-20T19:45:43,200 INFO  [Time-limited test {}] server.Server(415): Started @302274ms
2024-11-20T19:45:43,201 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:45:43,225 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T19:45:43,227 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T19:45:43,227 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T19:45:43,228 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T19:45:43,228 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T19:45:43,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f95849a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,AVAILABLE}
2024-11-20T19:45:43,228 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20dd8a9c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T19:45:43,325 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14628127{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/java.io.tmpdir/jetty-localhost-36805-hadoop-hdfs-3_4_1-tests_jar-_-any-3832816838853547896/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:43,325 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5982a43f{HTTP/1.1, (http/1.1)}{localhost:36805}
2024-11-20T19:45:43,325 INFO  [Time-limited test {}] server.Server(415): Started @302399ms
2024-11-20T19:45:43,326 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T19:45:43,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:43,956 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:44,315 WARN  [Thread-2513 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data1/current/BP-214577996-172.17.0.3-1732131942310/current, will proceed with Du for space computation calculation, 
2024-11-20T19:45:44,315 WARN  [Thread-2514 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data2/current/BP-214577996-172.17.0.3-1732131942310/current, will proceed with Du for space computation calculation, 
2024-11-20T19:45:44,331 WARN  [Thread-2477 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:45:44,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x395c61be9880428f with lease ID 0xf93015b97a13e99: Processing first storage report for DS-54f46cc6-8345-483c-815a-e698141515a8 from datanode DatanodeRegistration(127.0.0.1:44477, datanodeUuid=6d0a19da-546a-4134-bcbd-89f6abd5b64e, infoPort=39779, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310)
2024-11-20T19:45:44,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x395c61be9880428f with lease ID 0xf93015b97a13e99: from storage DS-54f46cc6-8345-483c-815a-e698141515a8 node DatanodeRegistration(127.0.0.1:44477, datanodeUuid=6d0a19da-546a-4134-bcbd-89f6abd5b64e, infoPort=39779, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T19:45:44,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x395c61be9880428f with lease ID 0xf93015b97a13e99: Processing first storage report for DS-7f8927da-d445-407a-9dd4-b4ca7ff2724b from datanode DatanodeRegistration(127.0.0.1:44477, datanodeUuid=6d0a19da-546a-4134-bcbd-89f6abd5b64e, infoPort=39779, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310)
2024-11-20T19:45:44,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x395c61be9880428f with lease ID 0xf93015b97a13e99: from storage DS-7f8927da-d445-407a-9dd4-b4ca7ff2724b node DatanodeRegistration(127.0.0.1:44477, datanodeUuid=6d0a19da-546a-4134-bcbd-89f6abd5b64e, infoPort=39779, infoSecurePort=0, ipcPort=38325, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:45:44,430 WARN  [Thread-2524 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data3/current/BP-214577996-172.17.0.3-1732131942310/current, will proceed with Du for space computation calculation, 
2024-11-20T19:45:44,430 WARN  [Thread-2525 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data4/current/BP-214577996-172.17.0.3-1732131942310/current, will proceed with Du for space computation calculation, 
2024-11-20T19:45:44,453 WARN  [Thread-2500 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T19:45:44,455 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x892f4c022efa8ab2 with lease ID 0xf93015b97a13e9a: Processing first storage report for DS-7dd4a8ab-4de1-47c9-9c79-18057e79e268 from datanode DatanodeRegistration(127.0.0.1:35109, datanodeUuid=6a26b016-6207-422c-932e-1175cc2fe82d, infoPort=46211, infoSecurePort=0, ipcPort=39187, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310)
2024-11-20T19:45:44,455 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x892f4c022efa8ab2 with lease ID 0xf93015b97a13e9a: from storage DS-7dd4a8ab-4de1-47c9-9c79-18057e79e268 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=6a26b016-6207-422c-932e-1175cc2fe82d, infoPort=46211, infoSecurePort=0, ipcPort=39187, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:45:44,456 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x892f4c022efa8ab2 with lease ID 0xf93015b97a13e9a: Processing first storage report for DS-a15a96e4-7d22-470d-b23f-fb77b206c907 from datanode DatanodeRegistration(127.0.0.1:35109, datanodeUuid=6a26b016-6207-422c-932e-1175cc2fe82d, infoPort=46211, infoSecurePort=0, ipcPort=39187, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310)
2024-11-20T19:45:44,456 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x892f4c022efa8ab2 with lease ID 0xf93015b97a13e9a: from storage DS-a15a96e4-7d22-470d-b23f-fb77b206c907 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=6a26b016-6207-422c-932e-1175cc2fe82d, infoPort=46211, infoSecurePort=0, ipcPort=39187, storageInfo=lv=-57;cid=testClusterID;nsid=1752904406;c=1732131942310), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T19:45:44,554 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52
2024-11-20T19:45:44,558 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/zookeeper_0, clientPort=52589, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T19:45:44,559 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52589
2024-11-20T19:45:44,559 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,561 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:45:44,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741825_1001 (size=7)
2024-11-20T19:45:44,570 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e with version=8
2024-11-20T19:45:44,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:43327/user/jenkins/test-data/c8eb62ae-03f4-8d7c-46f8-a74b8fbbaa1a/hbase-staging
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T19:45:44,572 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:45:44,573 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40507
2024-11-20T19:45:44,574 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40507 connecting to ZooKeeper ensemble=127.0.0.1:52589
2024-11-20T19:45:44,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405070x0, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:45:44,652 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40507-0x10159c7f4170000 connected
2024-11-20T19:45:44,735 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,737 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,739 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:44,739 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e, hbase.cluster.distributed=false
2024-11-20T19:45:44,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:45:44,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40507
2024-11-20T19:45:44,742 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40507
2024-11-20T19:45:44,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40507
2024-11-20T19:45:44,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40507
2024-11-20T19:45:44,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40507
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/e14ef07817f4:0 server-side Connection retries=45
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T19:45:44,763 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T19:45:44,764 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35275
2024-11-20T19:45:44,765 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35275 connecting to ZooKeeper ensemble=127.0.0.1:52589
2024-11-20T19:45:44,765 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,766 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352750x0, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T19:45:44,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35275-0x10159c7f4170001 connected
2024-11-20T19:45:44,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:44,778 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T19:45:44,778 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T19:45:44,779 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T19:45:44,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T19:45:44,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35275
2024-11-20T19:45:44,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35275
2024-11-20T19:45:44,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35275
2024-11-20T19:45:44,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35275
2024-11-20T19:45:44,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35275
2024-11-20T19:45:44,793 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e14ef07817f4:40507
2024-11-20T19:45:44,793 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e14ef07817f4,40507,1732131944572
2024-11-20T19:45:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:45:44,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:45:44,798 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e14ef07817f4,40507,1732131944572
2024-11-20T19:45:44,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T19:45:44,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,809 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T19:45:44,809 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e14ef07817f4,40507,1732131944572 from backup master directory
2024-11-20T19:45:44,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e14ef07817f4,40507,1732131944572
2024-11-20T19:45:44,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:45:44,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T19:45:44,819 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:45:44,819 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e14ef07817f4,40507,1732131944572
2024-11-20T19:45:44,825 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/hbase.id] with ID: 8256dfab-9635-43a5-ba75-cb29d902b967
2024-11-20T19:45:44,825 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/.tmp/hbase.id
2024-11-20T19:45:44,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:45:44,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741826_1002 (size=42)
2024-11-20T19:45:44,836 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/.tmp/hbase.id]:[hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/hbase.id]
2024-11-20T19:45:44,844 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:44,844 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T19:45:44,845 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T19:45:44,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:45:44,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741827_1003 (size=196)
2024-11-20T19:45:44,856 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T19:45:44,856 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T19:45:44,857 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:45:44,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:45:44,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741828_1004 (size=1189)
2024-11-20T19:45:44,866 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store
2024-11-20T19:45:44,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:45:44,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741829_1005 (size=34)
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:45:44,872 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:44,872 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:44,872 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131944872Disabling compacts and flushes for region at 1732131944872Disabling writes for close at 1732131944872Writing region close event to WAL at 1732131944872Closed at 1732131944872
2024-11-20T19:45:44,873 WARN  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/.initializing
2024-11-20T19:45:44,873 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/WALs/e14ef07817f4,40507,1732131944572
2024-11-20T19:45:44,875 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C40507%2C1732131944572, suffix=, logDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/WALs/e14ef07817f4,40507,1732131944572, archiveDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/oldWALs, maxLogs=10
2024-11-20T19:45:44,875 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C40507%2C1732131944572.1732131944875
2024-11-20T19:45:44,879 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/WALs/e14ef07817f4,40507,1732131944572/e14ef07817f4%2C40507%2C1732131944572.1732131944875
2024-11-20T19:45:44,882 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39779:39779),(127.0.0.1/127.0.0.1:46211:46211)]
2024-11-20T19:45:44,886 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:45:44,886 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:44,886 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,886 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,887 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:45:44,888 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T19:45:44,889 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:44,889 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:44,889 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:45:44,890 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T19:45:44,890 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:44,890 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:45:44,890 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:45:44,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T19:45:44,891 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:44,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:45:44,891 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T19:45:44,892 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T19:45:44,892 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:44,893 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T19:45:44,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,893 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,894 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,895 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,895 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,895 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T19:45:44,896 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T19:45:44,897 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:45:44,898 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870894, jitterRate=0.10740004479885101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T19:45:44,898 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732131944886Initializing all the Stores at 1732131944887 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131944887Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131944887Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131944887Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131944887Cleaning up temporary data from old regions at 1732131944895 (+8 ms)Region opened successfully at 1732131944898 (+3 ms)
2024-11-20T19:45:44,898 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T19:45:44,900 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5cfcc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:45:44,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T19:45:44,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T19:45:44,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T19:45:44,901 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T19:45:44,902 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T19:45:44,902 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T19:45:44,902 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T19:45:44,904 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T19:45:44,904 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T19:45:44,913 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T19:45:44,914 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T19:45:44,914 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T19:45:44,924 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T19:45:44,925 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T19:45:44,926 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T19:45:44,935 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T19:45:44,936 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T19:45:44,945 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T19:45:44,950 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T19:45:44,954 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:44,956 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T19:45:44,957 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,967 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e14ef07817f4,40507,1732131944572, sessionid=0x10159c7f4170000, setting cluster-up flag (Was=false)
2024-11-20T19:45:44,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:44,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,019 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T19:45:45,020 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,40507,1732131944572
2024-11-20T19:45:45,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,082 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T19:45:45,085 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e14ef07817f4,40507,1732131944572
2024-11-20T19:45:45,087 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T19:45:45,090 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T19:45:45,090 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T19:45:45,091 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T19:45:45,091 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e14ef07817f4,40507,1732131944572
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T19:45:45,094 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:45:45,094 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e14ef07817f4:0, corePoolSize=5, maxPoolSize=5
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e14ef07817f4:0, corePoolSize=10, maxPoolSize=10
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:45:45,095 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732131975096
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T19:45:45,096 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:45:45,096 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,096 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T19:45:45,097 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T19:45:45,097 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T19:45:45,097 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T19:45:45,097 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T19:45:45,097 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T19:45:45,097 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131945097,5,FailOnTimeoutGroup]
2024-11-20T19:45:45,098 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131945098,5,FailOnTimeoutGroup]
2024-11-20T19:45:45,098 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,098 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,098 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T19:45:45,098 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,098 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,098 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T19:45:45,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:45:45,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741831_1007 (size=1321)
2024-11-20T19:45:45,105 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T19:45:45,105 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e
2024-11-20T19:45:45,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:45:45,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741832_1008 (size=32)
2024-11-20T19:45:45,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:45,114 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:45:45,115 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:45:45,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,116 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,116 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:45:45,117 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:45:45,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,117 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,117 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:45:45,118 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:45:45,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,119 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,119 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:45:45,119 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:45:45,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,120 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:45:45,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740
2024-11-20T19:45:45,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740
2024-11-20T19:45:45,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:45:45,122 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:45:45,122 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:45:45,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:45:45,125 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T19:45:45,125 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=791971, jitterRate=0.007043406367301941}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732131945113Initializing all the Stores at 1732131945114 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945114Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945114Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131945114Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945114Cleaning up temporary data from old regions at 1732131945122 (+8 ms)Region opened successfully at 1732131945126 (+4 ms)
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:45:45,126 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:45:45,126 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:45:45,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131945126Disabling compacts and flushes for region at 1732131945126Disabling writes for close at 1732131945126Writing region close event to WAL at 1732131945126Closed at 1732131945126
2024-11-20T19:45:45,127 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:45:45,128 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T19:45:45,128 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T19:45:45,129 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:45:45,130 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T19:45:45,184 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(746): ClusterId : 8256dfab-9635-43a5-ba75-cb29d902b967
2024-11-20T19:45:45,184 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T19:45:45,229 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T19:45:45,229 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T19:45:45,243 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T19:45:45,243 DEBUG [RS:0;e14ef07817f4:35275 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7383c6a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e14ef07817f4/172.17.0.3:0
2024-11-20T19:45:45,261 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e14ef07817f4:35275
2024-11-20T19:45:45,261 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T19:45:45,261 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T19:45:45,261 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T19:45:45,262 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(2659): reportForDuty to master=e14ef07817f4,40507,1732131944572 with port=35275, startcode=1732131944763
2024-11-20T19:45:45,262 DEBUG [RS:0;e14ef07817f4:35275 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T19:45:45,264 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35031, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T19:45:45,264 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40507 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,264 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40507 {}] master.ServerManager(517): Registering regionserver=e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,266 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e
2024-11-20T19:45:45,266 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37221
2024-11-20T19:45:45,266 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T19:45:45,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:45:45,272 DEBUG [RS:0;e14ef07817f4:35275 {}] zookeeper.ZKUtil(111): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,272 WARN  [RS:0;e14ef07817f4:35275 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T19:45:45,272 INFO  [RS:0;e14ef07817f4:35275 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:45:45,272 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,273 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e14ef07817f4,35275,1732131944763]
2024-11-20T19:45:45,276 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T19:45:45,277 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T19:45:45,278 INFO  [RS:0;e14ef07817f4:35275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T19:45:45,278 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,278 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T19:45:45,279 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T19:45:45,279 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e14ef07817f4:0, corePoolSize=2, maxPoolSize=2
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,279 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,280 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,280 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e14ef07817f4:0, corePoolSize=1, maxPoolSize=1
2024-11-20T19:45:45,280 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:45:45,280 DEBUG [RS:0;e14ef07817f4:35275 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e14ef07817f4:0, corePoolSize=3, maxPoolSize=3
2024-11-20T19:45:45,280 WARN  [e14ef07817f4:40507 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,280 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,35275,1732131944763-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:45:45,296 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T19:45:45,297 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,35275,1732131944763-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,297 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,297 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.Replication(171): e14ef07817f4,35275,1732131944763 started
2024-11-20T19:45:45,309 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,309 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1482): Serving as e14ef07817f4,35275,1732131944763, RpcServer on e14ef07817f4/172.17.0.3:35275, sessionid=0x10159c7f4170001
2024-11-20T19:45:45,309 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T19:45:45,309 DEBUG [RS:0;e14ef07817f4:35275 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,309 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,35275,1732131944763'
2024-11-20T19:45:45,309 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e14ef07817f4,35275,1732131944763'
2024-11-20T19:45:45,310 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T19:45:45,311 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T19:45:45,311 DEBUG [RS:0;e14ef07817f4:35275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T19:45:45,311 INFO  [RS:0;e14ef07817f4:35275 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T19:45:45,311 INFO  [RS:0;e14ef07817f4:35275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T19:45:45,415 INFO  [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C35275%2C1732131944763, suffix=, logDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/e14ef07817f4,35275,1732131944763, archiveDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs, maxLogs=32
2024-11-20T19:45:45,416 INFO  [RS:0;e14ef07817f4:35275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C35275%2C1732131944763.1732131945416
2024-11-20T19:45:45,426 INFO  [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/e14ef07817f4,35275,1732131944763/e14ef07817f4%2C35275%2C1732131944763.1732131945416
2024-11-20T19:45:45,428 DEBUG [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39779:39779),(127.0.0.1/127.0.0.1:46211:46211)]
2024-11-20T19:45:45,530 DEBUG [e14ef07817f4:40507 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T19:45:45,531 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,532 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,35275,1732131944763, state=OPENING
2024-11-20T19:45:45,545 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T19:45:45,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,557 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T19:45:45,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:45:45,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:45:45,557 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,35275,1732131944763}]
2024-11-20T19:45:45,713 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T19:45:45,719 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59107, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T19:45:45,724 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T19:45:45,724 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:45:45,726 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e14ef07817f4%2C35275%2C1732131944763.meta, suffix=.meta, logDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/e14ef07817f4,35275,1732131944763, archiveDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs, maxLogs=32
2024-11-20T19:45:45,727 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor e14ef07817f4%2C35275%2C1732131944763.meta.1732131945726.meta
2024-11-20T19:45:45,731 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/e14ef07817f4,35275,1732131944763/e14ef07817f4%2C35275%2C1732131944763.meta.1732131945726.meta
2024-11-20T19:45:45,736 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39779:39779),(127.0.0.1/127.0.0.1:46211:46211)]
2024-11-20T19:45:45,740 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T19:45:45,741 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T19:45:45,741 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T19:45:45,742 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T19:45:45,743 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T19:45:45,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,743 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,744 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T19:45:45,744 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T19:45:45,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,744 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,745 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T19:45:45,745 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T19:45:45,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T19:45:45,746 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T19:45:45,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T19:45:45,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T19:45:45,747 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T19:45:45,747 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740
2024-11-20T19:45:45,748 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740
2024-11-20T19:45:45,749 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T19:45:45,749 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T19:45:45,749 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T19:45:45,750 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T19:45:45,751 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854056, jitterRate=0.08598895370960236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T19:45:45,751 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T19:45:45,751 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732131945741Writing region info on filesystem at 1732131945741Initializing all the Stores at 1732131945742 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945742Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945742Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732131945742Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732131945742Cleaning up temporary data from old regions at 1732131945749 (+7 ms)Running coprocessor post-open hooks at 1732131945751 (+2 ms)Region opened successfully at 1732131945751
2024-11-20T19:45:45,752 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732131945712
2024-11-20T19:45:45,754 DEBUG [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T19:45:45,754 INFO  [RS_OPEN_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T19:45:45,754 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,755 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e14ef07817f4,35275,1732131944763, state=OPEN
2024-11-20T19:45:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:45:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T19:45:45,829 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:45:45,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T19:45:45,834 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T19:45:45,834 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e14ef07817f4,35275,1732131944763 in 272 msec
2024-11-20T19:45:45,839 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T19:45:45,839 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 706 msec
2024-11-20T19:45:45,840 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T19:45:45,840 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T19:45:45,842 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:45:45,842 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,35275,1732131944763, seqNum=-1]
2024-11-20T19:45:45,843 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:45:45,844 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55825, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:45:45,851 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 761 msec
2024-11-20T19:45:45,852 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732131945852, completionTime=-1
2024-11-20T19:45:45,852 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T19:45:45,852 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-20T19:45:45,854 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-20T19:45:45,854 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732132005854
2024-11-20T19:45:45,854 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732132065854
2024-11-20T19:45:45,854 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e14ef07817f4:40507, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,855 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,857 DEBUG [master/e14ef07817f4:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T19:45:45,860 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.041sec
2024-11-20T19:45:45,860 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T19:45:45,860 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T19:45:45,861 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T19:45:45,861 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T19:45:45,861 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T19:45:45,861 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T19:45:45,861 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T19:45:45,864 DEBUG [master/e14ef07817f4:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T19:45:45,864 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T19:45:45,864 INFO  [master/e14ef07817f4:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e14ef07817f4,40507,1732131944572-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T19:45:45,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c62a047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:45:45,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e14ef07817f4,40507,-1 for getting cluster id
2024-11-20T19:45:45,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T19:45:45,885 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8256dfab-9635-43a5-ba75-cb29d902b967'
2024-11-20T19:45:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T19:45:45,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8256dfab-9635-43a5-ba75-cb29d902b967"

2024-11-20T19:45:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0eef60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:45:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e14ef07817f4,40507,-1]
2024-11-20T19:45:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T19:45:45,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:45,887 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T19:45:45,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ae05a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T19:45:45,888 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T19:45:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e14ef07817f4,35275,1732131944763, seqNum=-1]
2024-11-20T19:45:45,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T19:45:45,890 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T19:45:45,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e14ef07817f4,40507,1732131944572
2024-11-20T19:45:45,893 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T19:45:45,895 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T19:45:45,895 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T19:45:45,898 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs, maxLogs=32
2024-11-20T19:45:45,899 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732131945898
2024-11-20T19:45:45,904 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1/test.com%2C8080%2C1.1732131945898
2024-11-20T19:45:45,905 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46211:46211),(127.0.0.1/127.0.0.1:39779:39779)]
2024-11-20T19:45:45,906 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732131945905
2024-11-20T19:45:45,910 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,910 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,911 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,911 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,911 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,911 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1/test.com%2C8080%2C1.1732131945898 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1/test.com%2C8080%2C1.1732131945905
2024-11-20T19:45:45,912 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39779:39779),(127.0.0.1/127.0.0.1:46211:46211)]
2024-11-20T19:45:45,912 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1/test.com%2C8080%2C1.1732131945898 is not closed yet, will try archiving it next time
2024-11-20T19:45:45,913 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,913 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,913 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741835_1011 (size=93)
2024-11-20T19:45:45,913 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741835_1011 (size=93)
2024-11-20T19:45:45,914 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:45,915 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/WALs/test.com,8080,1/test.com%2C8080%2C1.1732131945898 to hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs/test.com%2C8080%2C1.1732131945898
2024-11-20T19:45:45,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741836_1012 (size=93)
2024-11-20T19:45:45,916 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741836_1012 (size=93)
2024-11-20T19:45:45,918 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs
2024-11-20T19:45:45,918 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732131945905)
2024-11-20T19:45:45,918 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T19:45:45,918 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:45:45,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:45,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:45,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:45,919 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T19:45:45,919 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T19:45:45,919 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=241299278, stopped=false
2024-11-20T19:45:45,919 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e14ef07817f4,40507,1732131944572
2024-11-20T19:45:45,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:45,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T19:45:45,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:45,935 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:45:45,935 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T19:45:45,935 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:45,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:45,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:45,936 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e14ef07817f4,35275,1732131944763' *****
2024-11-20T19:45:45,936 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T19:45:45,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T19:45:45,936 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T19:45:45,937 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T19:45:45,937 INFO  [RS:0;e14ef07817f4:35275 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T19:45:45,937 INFO  [RS:0;e14ef07817f4:35275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T19:45:45,937 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(959): stopping server e14ef07817f4,35275,1732131944763
2024-11-20T19:45:45,937 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:45:45,937 INFO  [RS:0;e14ef07817f4:35275 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e14ef07817f4:35275.
2024-11-20T19:45:45,937 DEBUG [RS:0;e14ef07817f4:35275 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T19:45:45,937 DEBUG [RS:0;e14ef07817f4:35275 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:45,938 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T19:45:45,938 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T19:45:45,938 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T19:45:45,938 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T19:45:45,938 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T19:45:45,938 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-20T19:45:45,938 DEBUG [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T19:45:45,938 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T19:45:45,939 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T19:45:45,939 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T19:45:45,939 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T19:45:45,939 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T19:45:45,939 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-20T19:45:45,955 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,41391,1732131753241/e14ef07817f4%2C41391%2C1732131753241.1732131753476
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:45,956 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/.tmp/ns/eaf2acfc9f2648f9a5257a74fbeb1aa1 is 43, key is default/ns:d/1732131945845/Put/seqid=0
2024-11-20T19:45:45,958 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:37581/user/jenkins/test-data/1834ad8d-5ad3-d803-242a-d7c370809b01/WALs/e14ef07817f4,37815,1732131751777/e14ef07817f4%2C37815%2C1732131751777.meta.1732131753012.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T19:45:45,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741837_1013 (size=5153)
2024-11-20T19:45:45,960 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741837_1013 (size=5153)
2024-11-20T19:45:45,960 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/.tmp/ns/eaf2acfc9f2648f9a5257a74fbeb1aa1
2024-11-20T19:45:45,965 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/.tmp/ns/eaf2acfc9f2648f9a5257a74fbeb1aa1 as hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/ns/eaf2acfc9f2648f9a5257a74fbeb1aa1
2024-11-20T19:45:45,969 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/ns/eaf2acfc9f2648f9a5257a74fbeb1aa1, entries=2, sequenceid=6, filesize=5.0 K
2024-11-20T19:45:45,970 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false
2024-11-20T19:45:45,970 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T19:45:45,974 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-20T19:45:45,974 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T19:45:45,974 INFO  [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T19:45:45,974 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732131945938Running coprocessor pre-close hooks at 1732131945938Disabling compacts and flushes for region at 1732131945938Disabling writes for close at 1732131945939 (+1 ms)Obtaining lock to block concurrent updates at 1732131945939Preparing flush snapshotting stores in 1588230740 at 1732131945939Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732131945939Flushing stores of hbase:meta,,1.1588230740 at 1732131945941 (+2 ms)Flushing 1588230740/ns: creating writer at 1732131945941Flushing 1588230740/ns: appending metadata at 1732131945955 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732131945955Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a1f2182: reopening flushed file at 1732131945964 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1732131945970 (+6 ms)Writing region close event to WAL at 1732131945971 (+1 ms)Running coprocessor post-close hooks at 1732131945974 (+3 ms)Closed at 1732131945974
2024-11-20T19:45:45,975 DEBUG [RS_CLOSE_META-regionserver/e14ef07817f4:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T19:45:46,139 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(976): stopping server e14ef07817f4,35275,1732131944763; all regions closed.
2024-11-20T19:45:46,140 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,140 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,141 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,141 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,141 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741834_1010 (size=1152)
2024-11-20T19:45:46,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741834_1010 (size=1152)
2024-11-20T19:45:46,151 DEBUG [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs
2024-11-20T19:45:46,151 INFO  [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C35275%2C1732131944763.meta:.meta(num 1732131945726)
2024-11-20T19:45:46,151 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,151 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,152 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,152 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,152 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741833_1009 (size=93)
2024-11-20T19:45:46,154 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741833_1009 (size=93)
2024-11-20T19:45:46,157 DEBUG [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/oldWALs
2024-11-20T19:45:46,157 INFO  [RS:0;e14ef07817f4:35275 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog e14ef07817f4%2C35275%2C1732131944763:(num 1732131945416)
2024-11-20T19:45:46,157 DEBUG [RS:0;e14ef07817f4:35275 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T19:45:46,157 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T19:45:46,157 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:45:46,157 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.ChoreService(370): Chore service for: regionserver/e14ef07817f4:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T19:45:46,157 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:45:46,157 INFO  [regionserver/e14ef07817f4:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:45:46,158 INFO  [RS:0;e14ef07817f4:35275 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35275
2024-11-20T19:45:46,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T19:45:46,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e14ef07817f4,35275,1732131944763
2024-11-20T19:45:46,166 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:45:46,177 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e14ef07817f4,35275,1732131944763]
2024-11-20T19:45:46,187 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e14ef07817f4,35275,1732131944763 already deleted, retry=false
2024-11-20T19:45:46,187 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e14ef07817f4,35275,1732131944763 expired; onlineServers=0
2024-11-20T19:45:46,187 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e14ef07817f4,40507,1732131944572' *****
2024-11-20T19:45:46,187 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T19:45:46,187 INFO  [M:0;e14ef07817f4:40507 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T19:45:46,188 INFO  [M:0;e14ef07817f4:40507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T19:45:46,188 DEBUG [M:0;e14ef07817f4:40507 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T19:45:46,188 DEBUG [M:0;e14ef07817f4:40507 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T19:45:46,188 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131945098 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.small.0-1732131945098,5,FailOnTimeoutGroup]
2024-11-20T19:45:46,188 DEBUG [master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131945097 {}] cleaner.HFileCleaner(306): Exit Thread[master/e14ef07817f4:0:becomeActiveMaster-HFileCleaner.large.0-1732131945097,5,FailOnTimeoutGroup]
2024-11-20T19:45:46,188 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T19:45:46,188 INFO  [M:0;e14ef07817f4:40507 {}] hbase.ChoreService(370): Chore service for: master/e14ef07817f4:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T19:45:46,188 INFO  [M:0;e14ef07817f4:40507 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T19:45:46,188 DEBUG [M:0;e14ef07817f4:40507 {}] master.HMaster(1795): Stopping service threads
2024-11-20T19:45:46,188 INFO  [M:0;e14ef07817f4:40507 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T19:45:46,188 INFO  [M:0;e14ef07817f4:40507 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T19:45:46,189 INFO  [M:0;e14ef07817f4:40507 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T19:45:46,189 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T19:45:46,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T19:45:46,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T19:45:46,198 DEBUG [M:0;e14ef07817f4:40507 {}] zookeeper.ZKUtil(347): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T19:45:46,198 WARN  [M:0;e14ef07817f4:40507 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T19:45:46,198 INFO  [M:0;e14ef07817f4:40507 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/.lastflushedseqids
2024-11-20T19:45:46,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741838_1014 (size=108)
2024-11-20T19:45:46,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741838_1014 (size=108)
2024-11-20T19:45:46,204 INFO  [M:0;e14ef07817f4:40507 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T19:45:46,204 INFO  [M:0;e14ef07817f4:40507 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T19:45:46,204 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T19:45:46,204 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:46,204 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:46,204 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T19:45:46,204 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:46,205 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-20T19:45:46,217 DEBUG [M:0;e14ef07817f4:40507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0d94debc822a44ae957d3c685004a5ed is 82, key is hbase:meta,,1/info:regioninfo/1732131945754/Put/seqid=0
2024-11-20T19:45:46,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741839_1015 (size=5672)
2024-11-20T19:45:46,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741839_1015 (size=5672)
2024-11-20T19:45:46,222 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0d94debc822a44ae957d3c685004a5ed
2024-11-20T19:45:46,239 DEBUG [M:0;e14ef07817f4:40507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1b8f08b708b4cbf93091a4d6282c87f is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732131945850/Put/seqid=0
2024-11-20T19:45:46,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741840_1016 (size=5275)
2024-11-20T19:45:46,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741840_1016 (size=5275)
2024-11-20T19:45:46,243 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1b8f08b708b4cbf93091a4d6282c87f
2024-11-20T19:45:46,260 DEBUG [M:0;e14ef07817f4:40507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49bc892cf7b0482198648815101513e4 is 69, key is e14ef07817f4,35275,1732131944763/rs:state/1732131945264/Put/seqid=0
2024-11-20T19:45:46,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741841_1017 (size=5156)
2024-11-20T19:45:46,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741841_1017 (size=5156)
2024-11-20T19:45:46,264 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49bc892cf7b0482198648815101513e4
2024-11-20T19:45:46,277 INFO  [RS:0;e14ef07817f4:35275 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:45:46,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:46,277 INFO  [RS:0;e14ef07817f4:35275 {}] regionserver.HRegionServer(1031): Exiting; stopping=e14ef07817f4,35275,1732131944763; zookeeper connection closed.
2024-11-20T19:45:46,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35275-0x10159c7f4170001, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:46,277 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@183e757d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@183e757d
2024-11-20T19:45:46,277 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T19:45:46,281 DEBUG [M:0;e14ef07817f4:40507 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/acfb211b9b144c7eacc4496fe10cff39 is 52, key is load_balancer_on/state:d/1732131945894/Put/seqid=0
2024-11-20T19:45:46,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741842_1018 (size=5056)
2024-11-20T19:45:46,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741842_1018 (size=5056)
2024-11-20T19:45:46,286 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/acfb211b9b144c7eacc4496fe10cff39
2024-11-20T19:45:46,290 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0d94debc822a44ae957d3c685004a5ed as hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0d94debc822a44ae957d3c685004a5ed
2024-11-20T19:45:46,294 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0d94debc822a44ae957d3c685004a5ed, entries=8, sequenceid=29, filesize=5.5 K
2024-11-20T19:45:46,295 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f1b8f08b708b4cbf93091a4d6282c87f as hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1b8f08b708b4cbf93091a4d6282c87f
2024-11-20T19:45:46,299 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f1b8f08b708b4cbf93091a4d6282c87f, entries=3, sequenceid=29, filesize=5.2 K
2024-11-20T19:45:46,300 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/49bc892cf7b0482198648815101513e4 as hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49bc892cf7b0482198648815101513e4
2024-11-20T19:45:46,304 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/49bc892cf7b0482198648815101513e4, entries=1, sequenceid=29, filesize=5.0 K
2024-11-20T19:45:46,305 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/acfb211b9b144c7eacc4496fe10cff39 as hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/acfb211b9b144c7eacc4496fe10cff39
2024-11-20T19:45:46,309 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37221/user/jenkins/test-data/08b5baae-d9b9-6218-f478-6a48f1b44c2e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/acfb211b9b144c7eacc4496fe10cff39, entries=1, sequenceid=29, filesize=4.9 K
2024-11-20T19:45:46,310 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=29, compaction requested=false
2024-11-20T19:45:46,311 INFO  [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T19:45:46,311 DEBUG [M:0;e14ef07817f4:40507 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732131946204Disabling compacts and flushes for region at 1732131946204Disabling writes for close at 1732131946204Obtaining lock to block concurrent updates at 1732131946205 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732131946205Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732131946205Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732131946205Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732131946206 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732131946217 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732131946217Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732131946226 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732131946238 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732131946238Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732131946247 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732131946259 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732131946259Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732131946267 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732131946281 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732131946281Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@592b12a4: reopening flushed file at 1732131946289 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30f35136: reopening flushed file at 1732131946294 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47dd370a: reopening flushed file at 1732131946299 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d30f2c8: reopening flushed file at 1732131946304 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=29, compaction requested=false at 1732131946310 (+6 ms)Writing region close event to WAL at 1732131946311 (+1 ms)Closed at 1732131946311
2024-11-20T19:45:46,312 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,312 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,312 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,312 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,312 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T19:45:46,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741830_1006 (size=10311)
2024-11-20T19:45:46,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44477 is added to blk_1073741830_1006 (size=10311)
2024-11-20T19:45:46,314 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T19:45:46,314 INFO  [M:0;e14ef07817f4:40507 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T19:45:46,314 INFO  [M:0;e14ef07817f4:40507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40507
2024-11-20T19:45:46,314 INFO  [M:0;e14ef07817f4:40507 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T19:45:46,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:46,424 INFO  [M:0;e14ef07817f4:40507 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T19:45:46,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40507-0x10159c7f4170000, quorum=127.0.0.1:52589, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T19:45:46,429 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14628127{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:46,430 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5982a43f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:46,430 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:46,430 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20dd8a9c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:46,430 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f95849a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:46,432 WARN  [BP-214577996-172.17.0.3-1732131942310 heartbeating to localhost/127.0.0.1:37221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:45:46,432 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:45:46,433 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:45:46,433 WARN  [BP-214577996-172.17.0.3-1732131942310 heartbeating to localhost/127.0.0.1:37221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-214577996-172.17.0.3-1732131942310 (Datanode Uuid 6a26b016-6207-422c-932e-1175cc2fe82d) service to localhost/127.0.0.1:37221
2024-11-20T19:45:46,433 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data3/current/BP-214577996-172.17.0.3-1732131942310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:46,433 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data4/current/BP-214577996-172.17.0.3-1732131942310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:46,434 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:45:46,435 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4bb82bb7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T19:45:46,436 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6a35d82b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:46,436 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:46,436 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ec9da7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:46,436 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1f1185ee{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:46,437 WARN  [BP-214577996-172.17.0.3-1732131942310 heartbeating to localhost/127.0.0.1:37221 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T19:45:46,437 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T19:45:46,437 WARN  [BP-214577996-172.17.0.3-1732131942310 heartbeating to localhost/127.0.0.1:37221 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-214577996-172.17.0.3-1732131942310 (Datanode Uuid 6d0a19da-546a-4134-bcbd-89f6abd5b64e) service to localhost/127.0.0.1:37221
2024-11-20T19:45:46,437 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T19:45:46,438 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data1/current/BP-214577996-172.17.0.3-1732131942310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:46,438 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/cluster_93574375-df98-83ca-62cd-09b5a3290cd1/data/data2/current/BP-214577996-172.17.0.3-1732131942310 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T19:45:46,438 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T19:45:46,443 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a7b167c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T19:45:46,444 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@735ef7ff{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T19:45:46,444 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T19:45:46,444 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a2ae1b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T19:45:46,444 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3910812a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a725a847-9894-3548-08e0-5344f571ac52/hadoop.log.dir/,STOPPED}
2024-11-20T19:45:46,451 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T19:45:46,464 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T19:45:46,472 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 230)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37221 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37221 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37221
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37221 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:37221
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37221
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=534 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=93 (was 57) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7149 (was 7162)