2024-11-20 06:45:17,364 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-20 06:45:17,380 main DEBUG Took 0.013602 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-20 06:45:17,381 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-20 06:45:17,381 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-20 06:45:17,383 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-20 06:45:17,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,394 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-20 06:45:17,412 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,415 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,416 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,418 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,420 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,421 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,423 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,424 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,425 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,426 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,427 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,428 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-20 06:45:17,429 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,430 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-20 06:45:17,431 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-20 06:45:17,433 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-20 06:45:17,436 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-20 06:45:17,437 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-20 06:45:17,438 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-20 06:45:17,439 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-20 06:45:17,451 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-20 06:45:17,455 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-20 06:45:17,457 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-20 06:45:17,458 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-20 06:45:17,458 main DEBUG createAppenders(={Console})
2024-11-20 06:45:17,459 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-20 06:45:17,460 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-20 06:45:17,460 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-20 06:45:17,461 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-20 06:45:17,461 main DEBUG OutputStream closed
2024-11-20 06:45:17,462 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-20 06:45:17,462 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-20 06:45:17,462 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-20 06:45:17,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-20 06:45:17,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-20 06:45:17,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-20 06:45:17,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-20 06:45:17,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-20 06:45:17,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-20 06:45:17,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-20 06:45:17,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-20 06:45:17,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-20 06:45:17,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-20 06:45:17,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-20 06:45:17,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-20 06:45:17,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-20 06:45:17,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-20 06:45:17,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-20 06:45:17,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-20 06:45:17,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-20 06:45:17,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-20 06:45:17,563 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-20 06:45:17,564 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-20 06:45:17,564 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-20 06:45:17,565 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-20T06:45:17,820 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425
2024-11-20 06:45:17,823 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-20 06:45:17,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-20T06:45:17,831 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-20T06:45:17,866 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=349, ProcessCount=11, AvailableMemoryMB=5317
2024-11-20T06:45:17,869 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:45:17,884 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85, deleteOnExit=true
2024-11-20T06:45:17,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:45:17,885 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/test.cache.data in system properties and HBase conf
2024-11-20T06:45:17,886 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:45:17,886 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:45:17,887 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:45:17,887 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:45:17,887 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:45:17,967 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-20T06:45:18,049 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:45:18,053 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:45:18,053 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:45:18,054 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:45:18,054 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:45:18,055 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:45:18,055 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:45:18,056 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:45:18,056 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:45:18,057 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:45:18,057 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:45:18,058 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:45:18,058 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:45:18,058 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:45:18,059 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:45:18,554 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:45:19,167 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-20T06:45:19,264 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2605ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-20T06:45:19,372 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:45:19,463 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:45:19,491 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:45:19,491 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:45:19,493 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:45:19,512 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:45:19,517 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64b8518a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:45:19,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22876862{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:45:19,766 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b31d0b7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/java.io.tmpdir/jetty-localhost-45541-hadoop-hdfs-3_4_1-tests_jar-_-any-9366103947446617035/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:45:19,774 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4fbd44e6{HTTP/1.1, (http/1.1)}{localhost:45541}
2024-11-20T06:45:19,774 INFO  [Time-limited test {}] server.Server(415): Started @3116ms
2024-11-20T06:45:19,808 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:45:20,394 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:45:20,401 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:45:20,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:45:20,403 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:45:20,404 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:45:20,404 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:45:20,405 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e9a9276{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:45:20,507 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27334b48{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/java.io.tmpdir/jetty-localhost-42685-hadoop-hdfs-3_4_1-tests_jar-_-any-15618381878384340802/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:45:20,507 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@502d300e{HTTP/1.1, (http/1.1)}{localhost:42685}
2024-11-20T06:45:20,508 INFO  [Time-limited test {}] server.Server(415): Started @3850ms
2024-11-20T06:45:20,559 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:45:20,684 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:45:20,689 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:45:20,695 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:45:20,695 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:45:20,696 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:45:20,698 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:45:20,699 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ba9a1a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:45:20,817 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58bedcbd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/java.io.tmpdir/jetty-localhost-38735-hadoop-hdfs-3_4_1-tests_jar-_-any-16727017399048469699/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:45:20,817 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74177ecc{HTTP/1.1, (http/1.1)}{localhost:38735}
2024-11-20T06:45:20,818 INFO  [Time-limited test {}] server.Server(415): Started @4160ms
2024-11-20T06:45:20,820 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:45:21,704 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data1/current/BP-617223157-172.17.0.2-1732085118654/current, will proceed with Du for space computation calculation, 
2024-11-20T06:45:21,704 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data2/current/BP-617223157-172.17.0.2-1732085118654/current, will proceed with Du for space computation calculation, 
2024-11-20T06:45:21,732 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:45:21,733 WARN  [Thread-104 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data3/current/BP-617223157-172.17.0.2-1732085118654/current, will proceed with Du for space computation calculation, 
2024-11-20T06:45:21,733 WARN  [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data4/current/BP-617223157-172.17.0.2-1732085118654/current, will proceed with Du for space computation calculation, 
2024-11-20T06:45:21,762 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:45:21,788 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11ae682072c952d8 with lease ID 0x95bc9ab6ba4c1119: Processing first storage report for DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67 from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=6f0f480b-2572-42b9-8f90-53e7b50d007a, infoPort=40743, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654)
2024-11-20T06:45:21,789 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11ae682072c952d8 with lease ID 0x95bc9ab6ba4c1119: from storage DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67 node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=6f0f480b-2572-42b9-8f90-53e7b50d007a, infoPort=40743, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:45:21,790 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79ca2525ad822aea with lease ID 0x95bc9ab6ba4c111a: Processing first storage report for DS-0b9c3088-2e66-4923-9da6-a39f74886de0 from datanode DatanodeRegistration(127.0.0.1:46455, datanodeUuid=a48bc62a-8a04-4fd2-818a-29ee9c147fc2, infoPort=34965, infoSecurePort=0, ipcPort=41207, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654)
2024-11-20T06:45:21,790 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79ca2525ad822aea with lease ID 0x95bc9ab6ba4c111a: from storage DS-0b9c3088-2e66-4923-9da6-a39f74886de0 node DatanodeRegistration(127.0.0.1:46455, datanodeUuid=a48bc62a-8a04-4fd2-818a-29ee9c147fc2, infoPort=34965, infoSecurePort=0, ipcPort=41207, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:45:21,790 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x11ae682072c952d8 with lease ID 0x95bc9ab6ba4c1119: Processing first storage report for DS-a7106abc-38ac-4a0d-b2c3-fdd6de820b50 from datanode DatanodeRegistration(127.0.0.1:39941, datanodeUuid=6f0f480b-2572-42b9-8f90-53e7b50d007a, infoPort=40743, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654)
2024-11-20T06:45:21,790 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11ae682072c952d8 with lease ID 0x95bc9ab6ba4c1119: from storage DS-a7106abc-38ac-4a0d-b2c3-fdd6de820b50 node DatanodeRegistration(127.0.0.1:39941, datanodeUuid=6f0f480b-2572-42b9-8f90-53e7b50d007a, infoPort=40743, infoSecurePort=0, ipcPort=46223, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:45:21,791 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x79ca2525ad822aea with lease ID 0x95bc9ab6ba4c111a: Processing first storage report for DS-b399b21d-4287-4b92-b43b-cc6b3ce22eab from datanode DatanodeRegistration(127.0.0.1:46455, datanodeUuid=a48bc62a-8a04-4fd2-818a-29ee9c147fc2, infoPort=34965, infoSecurePort=0, ipcPort=41207, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654)
2024-11-20T06:45:21,791 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x79ca2525ad822aea with lease ID 0x95bc9ab6ba4c111a: from storage DS-b399b21d-4287-4b92-b43b-cc6b3ce22eab node DatanodeRegistration(127.0.0.1:46455, datanodeUuid=a48bc62a-8a04-4fd2-818a-29ee9c147fc2, infoPort=34965, infoSecurePort=0, ipcPort=41207, storageInfo=lv=-57;cid=testClusterID;nsid=1215618520;c=1732085118654), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:45:21,834 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425
2024-11-20T06:45:21,918 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/zookeeper_0, clientPort=52366, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:45:21,930 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52366
2024-11-20T06:45:21,944 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:21,948 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:22,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:45:22,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:45:22,613 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d with version=8
2024-11-20T06:45:22,613 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:45:22,695 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-20T06:45:23,008 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:45:23,019 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,019 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,024 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:45:23,024 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,024 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:45:23,161 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:45:23,221 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-20T06:45:23,230 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-20T06:45:23,233 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:45:23,256 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22894 (auto-detected)
2024-11-20T06:45:23,258 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-20T06:45:23,279 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42589
2024-11-20T06:45:23,298 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42589 connecting to ZooKeeper ensemble=127.0.0.1:52366
2024-11-20T06:45:23,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:425890x0, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:45:23,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42589-0x1015847162a0000 connected
2024-11-20T06:45:23,716 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:23,718 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:23,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:45:23,730 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d, hbase.cluster.distributed=false
2024-11-20T06:45:23,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:45:23,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42589
2024-11-20T06:45:23,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42589
2024-11-20T06:45:23,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42589
2024-11-20T06:45:23,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42589
2024-11-20T06:45:23,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42589
2024-11-20T06:45:23,858 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:45:23,859 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,860 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,860 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:45:23,860 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:45:23,860 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:45:23,863 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:45:23,866 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:45:23,867 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46743
2024-11-20T06:45:23,870 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46743 connecting to ZooKeeper ensemble=127.0.0.1:52366
2024-11-20T06:45:23,872 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:23,878 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:23,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467430x0, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:45:23,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467430x0, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:45:23,906 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46743-0x1015847162a0001 connected
2024-11-20T06:45:23,909 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:45:23,917 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:45:23,919 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:45:23,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:45:23,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46743
2024-11-20T06:45:23,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46743
2024-11-20T06:45:23,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46743
2024-11-20T06:45:23,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46743
2024-11-20T06:45:23,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46743
2024-11-20T06:45:23,943 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:42589
2024-11-20T06:45:23,949 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:23,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:45:23,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:45:23,990 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:45:24,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,015 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:45:24,016 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,42589,1732085122817 from backup master directory
2024-11-20T06:45:24,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:45:24,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:45:24,023 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:45:24,024 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,026 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-20T06:45:24,027 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-20T06:45:24,092 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase.id] with ID: f14a3644-b07b-4718-b42a-1e60491dfa30
2024-11-20T06:45:24,092 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/.tmp/hbase.id
2024-11-20T06:45:24,104 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:45:24,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:45:24,106 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/.tmp/hbase.id]:[hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase.id]
2024-11-20T06:45:24,150 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:24,156 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:45:24,173 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-20T06:45:24,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:45:24,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:45:24,243 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:45:24,244 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:45:24,249 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:45:24,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:45:24,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:45:24,295 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store
2024-11-20T06:45:24,313 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:45:24,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:45:24,319 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-20T06:45:24,323 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:24,324 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:45:24,325 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:45:24,325 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:45:24,326 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:45:24,327 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:45:24,327 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:45:24,328 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085124324Disabling compacts and flushes for region at 1732085124324Disabling writes for close at 1732085124327 (+3 ms)Writing region close event to WAL at 1732085124327Closed at 1732085124327
2024-11-20T06:45:24,331 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/.initializing
2024-11-20T06:45:24,331 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/WALs/61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,353 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C42589%2C1732085122817, suffix=, logDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/WALs/61c2066a8a0f,42589,1732085122817, archiveDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/oldWALs, maxLogs=10
2024-11-20T06:45:24,364 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42589%2C1732085122817.1732085124359
2024-11-20T06:45:24,384 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/WALs/61c2066a8a0f,42589,1732085122817/61c2066a8a0f%2C42589%2C1732085122817.1732085124359
2024-11-20T06:45:24,392 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40743:40743),(127.0.0.1/127.0.0.1:34965:34965)]
2024-11-20T06:45:24,393 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:45:24,393 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:24,396 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,397 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,430 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:45:24,453 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:45:24,457 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:24,459 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:24,460 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:45:24,464 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:45:24,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:24,465 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:45:24,466 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:45:24,469 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:45:24,469 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:24,470 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:45:24,471 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:45:24,474 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:45:24,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:24,475 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:45:24,476 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,479 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,481 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,487 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,488 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:45:24,496 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:45:24,502 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:45:24,504 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719732, jitterRate=-0.08481359481811523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:45:24,514 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085124407Initializing all the Stores at 1732085124409 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085124410 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085124411 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085124411Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085124411Cleaning up temporary data from old regions at 1732085124488 (+77 ms)Region opened successfully at 1732085124514 (+26 ms)
2024-11-20T06:45:24,516 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:45:24,556 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7176d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:45:24,597 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:45:24,611 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:45:24,611 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:45:24,615 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:45:24,617 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-20T06:45:24,622 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-11-20T06:45:24,623 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:45:24,651 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:45:24,662 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:45:24,713 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:45:24,716 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:45:24,719 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:45:24,730 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:45:24,734 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:45:24,738 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:45:24,748 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:45:24,750 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:45:24,755 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:45:24,775 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:45:24,780 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:45:24,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:45:24,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:45:24,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,792 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,42589,1732085122817, sessionid=0x1015847162a0000, setting cluster-up flag (Was=false)
2024-11-20T06:45:24,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,847 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:45:24,850 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:24,897 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:45:24,901 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:24,908 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:45:24,932 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(746): ClusterId : f14a3644-b07b-4718-b42a-1e60491dfa30
2024-11-20T06:45:24,935 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:45:24,950 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:45:24,950 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:45:24,957 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:45:24,958 DEBUG [RS:0;61c2066a8a0f:46743 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89b1e2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:45:24,978 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:46743
2024-11-20T06:45:24,983 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:45:24,983 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:45:24,984 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:45:24,988 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,42589,1732085122817 with port=46743, startcode=1732085123822
2024-11-20T06:45:25,001 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:45:25,002 DEBUG [RS:0;61c2066a8a0f:46743 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:45:25,010 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:45:25,017 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:45:25,025 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,42589,1732085122817
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:45:25,034 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,035 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:45:25,035 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,039 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085155039
2024-11-20T06:45:25,041 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:45:25,041 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:45:25,042 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:45:25,043 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:45:25,047 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:45:25,047 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:45:25,048 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:45:25,048 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:45:25,049 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,049 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:45:25,074 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:45:25,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:45:25,087 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:45:25,096 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37213, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:45:25,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:45:25,102 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:45:25,102 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:45:25,103 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:45:25,107 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:45:25,107 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d
2024-11-20T06:45:25,105 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42589 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-20T06:45:25,111 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085125104,5,FailOnTimeoutGroup]
2024-11-20T06:45:25,112 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085125111,5,FailOnTimeoutGroup]
2024-11-20T06:45:25,112 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,112 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:45:25,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,127 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:45:25,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:45:25,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:25,133 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:45:25,135 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:45:25,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,137 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,137 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:45:25,140 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-11-20T06:45:25,140 WARN  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-11-20T06:45:25,140 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:45:25,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,142 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:45:25,145 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:45:25,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:45:25,148 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:45:25,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,149 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:45:25,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740
2024-11-20T06:45:25,152 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740
2024-11-20T06:45:25,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:45:25,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:45:25,157 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:45:25,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:45:25,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:45:25,166 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802503, jitterRate=0.020435631275177002}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:45:25,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085125130Initializing all the Stores at 1732085125132 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125132Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125133 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085125133Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125133Cleaning up temporary data from old regions at 1732085125156 (+23 ms)Region opened successfully at 1732085125170 (+14 ms)
2024-11-20T06:45:25,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:45:25,171 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:45:25,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:45:25,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:45:25,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:45:25,172 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:45:25,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085125171Disabling compacts and flushes for region at 1732085125171Disabling writes for close at 1732085125171Writing region close event to WAL at 1732085125172 (+1 ms)Closed at 1732085125172
2024-11-20T06:45:25,175 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:45:25,176 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:45:25,181 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:45:25,188 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:45:25,191 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:45:25,242 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,42589,1732085122817 with port=46743, startcode=1732085123822
2024-11-20T06:45:25,248 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42589 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,252 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42589 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,260 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d
2024-11-20T06:45:25,261 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33799
2024-11-20T06:45:25,261 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:45:25,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:45:25,289 DEBUG [RS:0;61c2066a8a0f:46743 {}] zookeeper.ZKUtil(111): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,289 WARN  [RS:0;61c2066a8a0f:46743 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:45:25,290 INFO  [RS:0;61c2066a8a0f:46743 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:45:25,290 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,293 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,46743,1732085123822]
2024-11-20T06:45:25,316 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:45:25,329 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:45:25,333 INFO  [RS:0;61c2066a8a0f:46743 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:45:25,334 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,334 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:45:25,339 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:45:25,340 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,340 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,340 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,341 WARN  [61c2066a8a0f:42589 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:45:25,341 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,342 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:45:25,342 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:45:25,342 DEBUG [RS:0;61c2066a8a0f:46743 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,343 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,46743,1732085123822-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:45:25,360 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:45:25,362 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,46743,1732085123822-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,362 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,362 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.Replication(171): 61c2066a8a0f,46743,1732085123822 started
2024-11-20T06:45:25,378 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:25,378 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,46743,1732085123822, RpcServer on 61c2066a8a0f/172.17.0.2:46743, sessionid=0x1015847162a0001
2024-11-20T06:45:25,379 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:45:25,379 DEBUG [RS:0;61c2066a8a0f:46743 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,379 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,46743,1732085123822'
2024-11-20T06:45:25,379 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:45:25,380 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:45:25,381 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:45:25,381 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:45:25,381 DEBUG [RS:0;61c2066a8a0f:46743 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,381 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,46743,1732085123822'
2024-11-20T06:45:25,381 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:45:25,382 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:45:25,383 DEBUG [RS:0;61c2066a8a0f:46743 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:45:25,383 INFO  [RS:0;61c2066a8a0f:46743 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:45:25,383 INFO  [RS:0;61c2066a8a0f:46743 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:45:25,495 INFO  [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C46743%2C1732085123822, suffix=, logDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822, archiveDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs, maxLogs=32
2024-11-20T06:45:25,497 INFO  [RS:0;61c2066a8a0f:46743 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085125497
2024-11-20T06:45:25,506 INFO  [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085125497
2024-11-20T06:45:25,508 DEBUG [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:45:25,595 DEBUG [61c2066a8a0f:42589 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:45:25,608 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,613 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,46743,1732085123822, state=OPENING
2024-11-20T06:45:25,647 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:45:25,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:25,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:45:25,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:45:25,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:45:25,659 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:45:25,664 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,46743,1732085123822}]
2024-11-20T06:45:25,851 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:45:25,854 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43331, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:45:25,864 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:45:25,865 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:45:25,869 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C46743%2C1732085123822.meta, suffix=.meta, logDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822, archiveDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs, maxLogs=32
2024-11-20T06:45:25,871 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.meta.1732085125871.meta
2024-11-20T06:45:25,878 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.meta.1732085125871.meta
2024-11-20T06:45:25,883 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:45:25,884 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:45:25,885 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:45:25,888 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:45:25,892 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:45:25,896 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:45:25,896 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:25,897 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:45:25,897 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:45:25,900 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:45:25,902 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:45:25,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,903 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,903 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:45:25,905 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:45:25,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,907 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,907 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:45:25,909 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:45:25,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,910 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,910 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:45:25,912 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:45:25,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:25,913 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:45:25,913 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:45:25,915 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740
2024-11-20T06:45:25,917 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740
2024-11-20T06:45:25,920 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:45:25,920 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:45:25,921 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:45:25,924 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:45:25,926 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831441, jitterRate=0.05723308026790619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:45:25,926 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:45:25,928 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085125897Writing region info on filesystem at 1732085125897Initializing all the Stores at 1732085125899 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125899Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125900 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085125900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085125900Cleaning up temporary data from old regions at 1732085125920 (+20 ms)Running coprocessor post-open hooks at 1732085125926 (+6 ms)Region opened successfully at 1732085125928 (+2 ms)
2024-11-20T06:45:25,933 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085125842
2024-11-20T06:45:25,946 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:25,948 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:45:25,948 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:45:25,949 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,46743,1732085123822, state=OPEN
2024-11-20T06:45:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:45:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:45:26,005 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:45:26,005 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:45:26,006 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:26,012 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:45:26,012 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,46743,1732085123822 in 343 msec
2024-11-20T06:45:26,019 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:45:26,019 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 833 msec
2024-11-20T06:45:26,021 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:45:26,022 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:45:26,040 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:45:26,041 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,46743,1732085123822, seqNum=-1]
2024-11-20T06:45:26,061 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:45:26,063 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:45:26,083 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1390 sec
2024-11-20T06:45:26,083 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085126083, completionTime=-1
2024-11-20T06:45:26,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:45:26,086 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:45:26,108 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:45:26,108 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085186108
2024-11-20T06:45:26,108 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085246108
2024-11-20T06:45:26,108 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec
2024-11-20T06:45:26,111 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,111 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,111 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:42589, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,120 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:45:26,139 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.115sec
2024-11-20T06:45:26,140 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:45:26,141 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:45:26,142 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:45:26,142 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:45:26,143 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:45:26,143 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:45:26,144 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:45:26,152 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:45:26,153 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:45:26,153 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42589,1732085122817-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:45:26,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e964458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:45:26,245 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-20T06:45:26,245 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-20T06:45:26,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,42589,-1 for getting cluster id
2024-11-20T06:45:26,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:45:26,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f14a3644-b07b-4718-b42a-1e60491dfa30'
2024-11-20T06:45:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:45:26,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f14a3644-b07b-4718-b42a-1e60491dfa30"

2024-11-20T06:45:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3741d59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:45:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,42589,-1]
2024-11-20T06:45:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:45:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:45:26,273 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:45:26,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500f1e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:45:26,277 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,46743,1732085123822, seqNum=-1]
2024-11-20T06:45:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:45:26,287 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:45:26,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:26,305 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:45:26,311 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:45:26,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T06:45:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,42589,1732085122817
2024-11-20T06:45:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d
2024-11-20T06:45:26,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T06:45:26,327 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T06:45:26,329 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T06:45:26,329 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T06:45:26,333 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:45:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-20T06:45:26,360 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T06:45:26,362 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-20T06:45:26,362 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:26,365 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T06:45:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:45:26,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741835_1011 (size=389)
2024-11-20T06:45:26,427 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741835_1011 (size=389)
2024-11-20T06:45:26,431 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 03e86be360b2cee8e4aca8882a8efd3d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d
2024-11-20T06:45:26,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741836_1012 (size=72)
2024-11-20T06:45:26,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741836_1012 (size=72)
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 03e86be360b2cee8e4aca8882a8efd3d, disabling compactions & flushes
2024-11-20T06:45:26,444 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d. after waiting 0 ms
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,444 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,444 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 03e86be360b2cee8e4aca8882a8efd3d:
Waiting for close lock at 1732085126444Disabling compacts and flushes for region at 1732085126444Disabling writes for close at 1732085126444Writing region close event to WAL at 1732085126444Closed at 1732085126444
2024-11-20T06:45:26,446 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T06:45:26,451 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732085126446"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085126446"}]},"ts":"1732085126446"}
2024-11-20T06:45:26,455 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T06:45:26,457 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T06:45:26,459 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085126457"}]},"ts":"1732085126457"}
2024-11-20T06:45:26,464 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-20T06:45:26,465 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=03e86be360b2cee8e4aca8882a8efd3d, ASSIGN}]
2024-11-20T06:45:26,468 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=03e86be360b2cee8e4aca8882a8efd3d, ASSIGN
2024-11-20T06:45:26,470 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=03e86be360b2cee8e4aca8882a8efd3d, ASSIGN; state=OFFLINE, location=61c2066a8a0f,46743,1732085123822; forceNewPlan=false, retain=false
2024-11-20T06:45:26,622 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03e86be360b2cee8e4aca8882a8efd3d, regionState=OPENING, regionLocation=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:26,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=03e86be360b2cee8e4aca8882a8efd3d, ASSIGN because future has completed
2024-11-20T06:45:26,633 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e86be360b2cee8e4aca8882a8efd3d, server=61c2066a8a0f,46743,1732085123822}]
2024-11-20T06:45:26,802 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,802 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 03e86be360b2cee8e4aca8882a8efd3d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:45:26,803 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,803 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:45:26,803 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,803 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,805 INFO  [StoreOpener-03e86be360b2cee8e4aca8882a8efd3d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 03e86be360b2cee8e4aca8882a8efd3d 
2024-11-20T06:45:26,808 INFO  [StoreOpener-03e86be360b2cee8e4aca8882a8efd3d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03e86be360b2cee8e4aca8882a8efd3d columnFamilyName info
2024-11-20T06:45:26,808 DEBUG [StoreOpener-03e86be360b2cee8e4aca8882a8efd3d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:45:26,810 INFO  [StoreOpener-03e86be360b2cee8e4aca8882a8efd3d-1 {}] regionserver.HStore(327): Store=03e86be360b2cee8e4aca8882a8efd3d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:45:26,810 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,811 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,812 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,812 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,812 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,815 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,818 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:45:26,819 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 03e86be360b2cee8e4aca8882a8efd3d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=878350, jitterRate=0.11688074469566345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:45:26,819 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:26,820 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 03e86be360b2cee8e4aca8882a8efd3d:
Running coprocessor pre-open hook at 1732085126803Writing region info on filesystem at 1732085126803Initializing all the Stores at 1732085126805 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085126805Cleaning up temporary data from old regions at 1732085126812 (+7 ms)Running coprocessor post-open hooks at 1732085126819 (+7 ms)Region opened successfully at 1732085126820 (+1 ms)
2024-11-20T06:45:26,823 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d., pid=6, masterSystemTime=1732085126788
2024-11-20T06:45:26,826 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,827 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:26,828 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=03e86be360b2cee8e4aca8882a8efd3d, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,46743,1732085123822
2024-11-20T06:45:26,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e86be360b2cee8e4aca8882a8efd3d, server=61c2066a8a0f,46743,1732085123822 because future has completed
2024-11-20T06:45:26,839 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T06:45:26,840 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 03e86be360b2cee8e4aca8882a8efd3d, server=61c2066a8a0f,46743,1732085123822 in 201 msec
2024-11-20T06:45:26,843 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T06:45:26,843 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=03e86be360b2cee8e4aca8882a8efd3d, ASSIGN in 374 msec
2024-11-20T06:45:26,844 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T06:45:26,845 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085126844"}]},"ts":"1732085126844"}
2024-11-20T06:45:26,847 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-20T06:45:26,849 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T06:45:26,853 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 514 msec
2024-11-20T06:45:31,411 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-20T06:45:31,473 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:45:31,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-20T06:45:33,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:45:33,219 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T06:45:33,223 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-20T06:45:33,223 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T06:45:33,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:45:33,225 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T06:45:33,226 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T06:45:33,226 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-20T06:45:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42589 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:45:36,475 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-20T06:45:36,479 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-20T06:45:36,485 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-20T06:45:36,486 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:45:36,487 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085136487
2024-11-20T06:45:36,495 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:36,495 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:36,495 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:36,495 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:36,495 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:36,496 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085125497 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085136487
2024-11-20T06:45:36,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:45:36,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085125497 is not closed yet, will try archiving it next time
2024-11-20T06:45:36,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741833_1009 (size=451)
2024-11-20T06:45:36,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741833_1009 (size=451)
2024-11-20T06:45:36,501 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085125497 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085125497
2024-11-20T06:45:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d., hostname=61c2066a8a0f,46743,1732085123822, seqNum=2]
2024-11-20T06:45:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46743 {}] regionserver.HRegion(8855): Flush requested on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:45:48,548 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03e86be360b2cee8e4aca8882a8efd3d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:45:48,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/d12a8940c2034715a0ad13d965df975e is 1080, key is row0001/info:/1732085136510/Put/seqid=0
2024-11-20T06:45:48,635 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741838_1014 (size=12509)
2024-11-20T06:45:48,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741838_1014 (size=12509)
2024-11-20T06:45:48,641 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/d12a8940c2034715a0ad13d965df975e
2024-11-20T06:45:48,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/d12a8940c2034715a0ad13d965df975e as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e
2024-11-20T06:45:48,728 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e, entries=7, sequenceid=11, filesize=12.2 K
2024-11-20T06:45:48,738 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 187ms, sequenceid=11, compaction requested=false
2024-11-20T06:45:48,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03e86be360b2cee8e4aca8882a8efd3d:

2024-11-20T06:45:51,830 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:45:56,559 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085156558
2024-11-20T06:45:56,768 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:45:56,768 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:56,768 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:56,768 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:56,768 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:56,768 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:45:56,769 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085136487 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085156558
2024-11-20T06:45:56,769 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:45:56,769 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085136487 is not closed yet, will try archiving it next time
2024-11-20T06:45:56,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741837_1013 (size=12399)
2024-11-20T06:45:56,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741837_1013 (size=12399)
2024-11-20T06:45:56,973 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:45:59,180 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:01,386 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:03,591 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46743 {}] regionserver.HRegion(8855): Flush requested on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:03,592 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03e86be360b2cee8e4aca8882a8efd3d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:46:03,794 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:03,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/ac58005a6e254c09a2987d3ccf2ac8f5 is 1080, key is row0008/info:/1732085150547/Put/seqid=0
2024-11-20T06:46:03,825 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741840_1016 (size=12509)
2024-11-20T06:46:03,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741840_1016 (size=12509)
2024-11-20T06:46:03,826 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/ac58005a6e254c09a2987d3ccf2ac8f5
2024-11-20T06:46:03,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/ac58005a6e254c09a2987d3ccf2ac8f5 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5
2024-11-20T06:46:03,853 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5, entries=7, sequenceid=21, filesize=12.2 K
2024-11-20T06:46:04,055 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:04,055 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 464ms, sequenceid=21, compaction requested=false
2024-11-20T06:46:04,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03e86be360b2cee8e4aca8882a8efd3d:

2024-11-20T06:46:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-20T06:46:04,056 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:04,057 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e because midkey is the same as first or last row
2024-11-20T06:46:05,796 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:06,667 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T06:46:06,667 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T06:46:08,002 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:08,005 WARN  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:08,006 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C46743%2C1732085123822:(num 1732085156558) roll requested
2024-11-20T06:46:08,007 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085168006
2024-11-20T06:46:08,215 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:08,215 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:08,215 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:08,215 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:08,216 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:08,216 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:08,216 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085156558 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085168006
2024-11-20T06:46:08,218 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:46:08,218 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085156558 is not closed yet, will try archiving it next time
2024-11-20T06:46:08,219 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085136487 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085136487
2024-11-20T06:46:08,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741839_1015 (size=7739)
2024-11-20T06:46:08,219 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741839_1015 (size=7739)
2024-11-20T06:46:10,208 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:11,803 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 03e86be360b2cee8e4aca8882a8efd3d, had cached 0 bytes from a total of 25018
2024-11-20T06:46:12,412 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:14,618 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:16,824 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:18,827 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T06:46:18,828 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085178828
2024-11-20T06:46:21,830 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:46:23,843 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:23,846 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:23,846 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C46743%2C1732085123822:(num 1732085178828) roll requested
2024-11-20T06:46:23,849 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:23,849 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:23,849 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:23,849 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:23,850 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:23,850 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085168006 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085178828
2024-11-20T06:46:23,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741841_1017 (size=4753)
2024-11-20T06:46:23,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741841_1017 (size=4753)
2024-11-20T06:46:23,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:46:23,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085168006 is not closed yet, will try archiving it next time
2024-11-20T06:46:23,860 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085183860
2024-11-20T06:46:28,863 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:28,863 WARN  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46743 {}] regionserver.HRegion(8855): Flush requested on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:28,864 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03e86be360b2cee8e4aca8882a8efd3d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:46:28,876 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:28,876 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:30,865 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T06:46:33,866 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:33,866 WARN  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK], DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK]]
2024-11-20T06:46:33,866 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:33,866 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:33,866 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:33,866 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:33,866 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:33,867 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085178828 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085183860
2024-11-20T06:46:33,868 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40743:40743),(127.0.0.1/127.0.0.1:34965:34965)]
2024-11-20T06:46:33,869 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085178828 is not closed yet, will try archiving it next time
2024-11-20T06:46:33,869 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C46743%2C1732085123822:(num 1732085183860) roll requested
2024-11-20T06:46:33,869 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085193869
2024-11-20T06:46:33,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741842_1018 (size=1569)
2024-11-20T06:46:33,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741842_1018 (size=1569)
2024-11-20T06:46:33,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/dc70827c387449649526965472d1483a is 1080, key is row0015/info:/1732085165593/Put/seqid=0
2024-11-20T06:46:33,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741844_1020 (size=12509)
2024-11-20T06:46:33,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741844_1020 (size=12509)
2024-11-20T06:46:33,886 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/dc70827c387449649526965472d1483a
2024-11-20T06:46:33,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/dc70827c387449649526965472d1483a as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a
2024-11-20T06:46:33,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a, entries=7, sequenceid=31, filesize=12.2 K
2024-11-20T06:46:38,884 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5012 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK], DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK]]
2024-11-20T06:46:38,884 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5012 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK], DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK]]
2024-11-20T06:46:38,915 INFO  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK], DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK]]
2024-11-20T06:46:38,915 WARN  [FSHLog-0-hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d-prefix:61c2066a8a0f,46743,1732085123822 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39941,DS-7023f8cf-2f12-4bbb-b95a-8ea0a57eec67,DISK], DatanodeInfoWithStorage[127.0.0.1:46455,DS-0b9c3088-2e66-4923-9da6-a39f74886de0,DISK]]
2024-11-20T06:46:38,915 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 10052ms, sequenceid=31, compaction requested=true
2024-11-20T06:46:38,915 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03e86be360b2cee8e4aca8882a8efd3d:

2024-11-20T06:46:38,915 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,915 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-20T06:46:38,916 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:38,916 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,916 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e because midkey is the same as first or last row
2024-11-20T06:46:38,916 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085183860 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085193869
2024-11-20T06:46:38,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741843_1019 (size=438)
2024-11-20T06:46:38,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741843_1019 (size=438)
2024-11-20T06:46:38,919 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085156558 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085156558
2024-11-20T06:46:38,921 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085168006 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085168006
2024-11-20T06:46:38,922 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085178828 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085178828
2024-11-20T06:46:38,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 03e86be360b2cee8e4aca8882a8efd3d:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:46:38,924 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085183860 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085183860
2024-11-20T06:46:38,924 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:46:38,924 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C46743%2C1732085123822:(num 1732085198924) roll requested
2024-11-20T06:46:38,924 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085198924
2024-11-20T06:46:38,925 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:46:38,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:46:38,929 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:46:38,931 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HStore(1541): 03e86be360b2cee8e4aca8882a8efd3d/info is initiating minor compaction (all files)
2024-11-20T06:46:38,932 INFO  [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 03e86be360b2cee8e4aca8882a8efd3d/info in TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:38,932 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,932 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,932 INFO  [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a] into tmpdir=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp, totalSize=36.6 K
2024-11-20T06:46:38,932 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,932 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,932 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,932 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085193869 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085198924
2024-11-20T06:46:38,934 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] compactions.Compactor(225): Compacting d12a8940c2034715a0ad13d965df975e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732085136510
2024-11-20T06:46:38,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741845_1021 (size=93)
2024-11-20T06:46:38,935 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac58005a6e254c09a2987d3ccf2ac8f5, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732085150547
2024-11-20T06:46:38,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741845_1021 (size=93)
2024-11-20T06:46:38,936 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085193869 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs/61c2066a8a0f%2C46743%2C1732085123822.1732085193869
2024-11-20T06:46:38,936 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc70827c387449649526965472d1483a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732085165593
2024-11-20T06:46:38,939 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34965:34965),(127.0.0.1/127.0.0.1:40743:40743)]
2024-11-20T06:46:38,940 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C46743%2C1732085123822.1732085198940
2024-11-20T06:46:38,952 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,952 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,952 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,952 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,952 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:38,953 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085198924 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085198940
2024-11-20T06:46:38,953 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40743:40743),(127.0.0.1/127.0.0.1:34965:34965)]
2024-11-20T06:46:38,953 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/WALs/61c2066a8a0f,46743,1732085123822/61c2066a8a0f%2C46743%2C1732085123822.1732085198924 is not closed yet, will try archiving it next time
2024-11-20T06:46:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741846_1022 (size=1258)
2024-11-20T06:46:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741846_1022 (size=1258)
2024-11-20T06:46:38,976 INFO  [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 03e86be360b2cee8e4aca8882a8efd3d#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:46:38,977 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/5b6e9d94b6804dc297ab4cc1d200465f is 1080, key is row0001/info:/1732085136510/Put/seqid=0
2024-11-20T06:46:38,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741848_1024 (size=27710)
2024-11-20T06:46:38,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741848_1024 (size=27710)
2024-11-20T06:46:39,397 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/5b6e9d94b6804dc297ab4cc1d200465f as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5b6e9d94b6804dc297ab4cc1d200465f
2024-11-20T06:46:39,413 INFO  [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 03e86be360b2cee8e4aca8882a8efd3d/info of 03e86be360b2cee8e4aca8882a8efd3d into 5b6e9d94b6804dc297ab4cc1d200465f(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:46:39,413 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 03e86be360b2cee8e4aca8882a8efd3d:

2024-11-20T06:46:39,415 INFO  [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d., storeName=03e86be360b2cee8e4aca8882a8efd3d/info, priority=13, startTime=1732085198917; duration=0sec
2024-11-20T06:46:39,415 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T06:46:39,415 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:39,415 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5b6e9d94b6804dc297ab4cc1d200465f because midkey is the same as first or last row
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5b6e9d94b6804dc297ab4cc1d200465f because midkey is the same as first or last row
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5b6e9d94b6804dc297ab4cc1d200465f because midkey is the same as first or last row
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:46:39,416 DEBUG [RS:0;61c2066a8a0f:46743-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 03e86be360b2cee8e4aca8882a8efd3d:info
2024-11-20T06:46:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46743 {}] regionserver.HRegion(8855): Flush requested on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:50,968 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 03e86be360b2cee8e4aca8882a8efd3d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:46:50,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/5f9d1c1b058b4aa9bbfc2b3659472e71 is 1080, key is row0022/info:/1732085198942/Put/seqid=0
2024-11-20T06:46:50,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741849_1025 (size=12509)
2024-11-20T06:46:50,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741849_1025 (size=12509)
2024-11-20T06:46:51,384 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/5f9d1c1b058b4aa9bbfc2b3659472e71
2024-11-20T06:46:51,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/5f9d1c1b058b4aa9bbfc2b3659472e71 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5f9d1c1b058b4aa9bbfc2b3659472e71
2024-11-20T06:46:51,406 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5f9d1c1b058b4aa9bbfc2b3659472e71, entries=7, sequenceid=42, filesize=12.2 K
2024-11-20T06:46:51,408 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 440ms, sequenceid=42, compaction requested=false
2024-11-20T06:46:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 03e86be360b2cee8e4aca8882a8efd3d:

2024-11-20T06:46:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-20T06:46:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:46:51,408 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/5b6e9d94b6804dc297ab4cc1d200465f because midkey is the same as first or last row
2024-11-20T06:46:51,831 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:46:56,804 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 03e86be360b2cee8e4aca8882a8efd3d, had cached 0 bytes from a total of 40219
2024-11-20T06:46:59,031 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:46:59,032 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:46:59,032 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:46:59,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:46:59,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:46:59,041 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:46:59,041 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:46:59,041 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=233511752, stopped=false
2024-11-20T06:46:59,042 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,42589,1732085122817
2024-11-20T06:46:59,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:46:59,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:46:59,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:46:59,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:46:59,076 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:46:59,076 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:46:59,076 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:46:59,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:46:59,077 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:46:59,077 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:46:59,077 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,46743,1732085123822' *****
2024-11-20T06:46:59,077 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:46:59,077 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:46:59,078 INFO  [RS:0;61c2066a8a0f:46743 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:46:59,078 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:46:59,078 INFO  [RS:0;61c2066a8a0f:46743 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:46:59,078 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(3091): Received CLOSE for 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,46743,1732085123822
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:46743.
2024-11-20T06:46:59,079 DEBUG [RS:0;61c2066a8a0f:46743 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:46:59,079 DEBUG [RS:0;61c2066a8a0f:46743 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:46:59,079 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03e86be360b2cee8e4aca8882a8efd3d, disabling compactions & flushes
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:46:59,079 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:46:59,079 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:59,079 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d. after waiting 0 ms
2024-11-20T06:46:59,079 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:46:59,079 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:59,079 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 03e86be360b2cee8e4aca8882a8efd3d 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-20T06:46:59,080 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T06:46:59,080 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:46:59,080 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:46:59,080 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1325): Online Regions={03e86be360b2cee8e4aca8882a8efd3d=TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:46:59,080 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:46:59,080 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:46:59,080 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:46:59,080 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-20T06:46:59,080 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1351): Waiting on 03e86be360b2cee8e4aca8882a8efd3d, 1588230740
2024-11-20T06:46:59,086 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/a5e9bc54284645bf89143893d862cc6e is 1080, key is row0029/info:/1732085212970/Put/seqid=0
2024-11-20T06:46:59,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741850_1026 (size=8193)
2024-11-20T06:46:59,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741850_1026 (size=8193)
2024-11-20T06:46:59,105 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/info/d568ccd7f33842d9a7391ee493d649a6 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d./info:regioninfo/1732085126827/Put/seqid=0
2024-11-20T06:46:59,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741851_1027 (size=7016)
2024-11-20T06:46:59,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741851_1027 (size=7016)
2024-11-20T06:46:59,112 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/info/d568ccd7f33842d9a7391ee493d649a6
2024-11-20T06:46:59,141 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/ns/324d2a6688f042ae8b38d50ab2a5bbf1 is 43, key is default/ns:d/1732085126067/Put/seqid=0
2024-11-20T06:46:59,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741852_1028 (size=5153)
2024-11-20T06:46:59,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741852_1028 (size=5153)
2024-11-20T06:46:59,150 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/ns/324d2a6688f042ae8b38d50ab2a5bbf1
2024-11-20T06:46:59,181 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/table/7fa5b5deff354655a6718fe601a9f390 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732085126844/Put/seqid=0
2024-11-20T06:46:59,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741853_1029 (size=5396)
2024-11-20T06:46:59,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741853_1029 (size=5396)
2024-11-20T06:46:59,190 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/table/7fa5b5deff354655a6718fe601a9f390
2024-11-20T06:46:59,200 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/info/d568ccd7f33842d9a7391ee493d649a6 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/info/d568ccd7f33842d9a7391ee493d649a6
2024-11-20T06:46:59,211 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/info/d568ccd7f33842d9a7391ee493d649a6, entries=10, sequenceid=11, filesize=6.9 K
2024-11-20T06:46:59,212 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/ns/324d2a6688f042ae8b38d50ab2a5bbf1 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/ns/324d2a6688f042ae8b38d50ab2a5bbf1
2024-11-20T06:46:59,223 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/ns/324d2a6688f042ae8b38d50ab2a5bbf1, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T06:46:59,224 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/.tmp/table/7fa5b5deff354655a6718fe601a9f390 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/table/7fa5b5deff354655a6718fe601a9f390
2024-11-20T06:46:59,234 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/table/7fa5b5deff354655a6718fe601a9f390, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T06:46:59,235 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 155ms, sequenceid=11, compaction requested=false
2024-11-20T06:46:59,242 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T06:46:59,245 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:46:59,245 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:46:59,245 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085219080Running coprocessor pre-close hooks at 1732085219080Disabling compacts and flushes for region at 1732085219080Disabling writes for close at 1732085219080Obtaining lock to block concurrent updates at 1732085219080Preparing flush snapshotting stores in 1588230740 at 1732085219080Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732085219081 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085219082 (+1 ms)Flushing 1588230740/info: creating writer at 1732085219082Flushing 1588230740/info: appending metadata at 1732085219104 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732085219104Flushing 1588230740/ns: creating writer at 1732085219123 (+19 ms)Flushing 1588230740/ns: appending metadata at 1732085219140 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732085219141 (+1 ms)Flushing 1588230740/table: creating writer at 1732085219159 (+18 ms)Flushing 1588230740/table: appending metadata at 1732085219180 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732085219181 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f45e7b3: reopening flushed file at 1732085219199 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d81d0a6: reopening flushed file at 1732085219211 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@618c4785: reopening flushed file at 1732085219223 (+12 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 155ms, sequenceid=11, compaction requested=false at 1732085219235 (+12 ms)Writing region close event to WAL at 1732085219237 (+2 ms)Running coprocessor post-close hooks at 1732085219243 (+6 ms)Closed at 1732085219245 (+2 ms)
2024-11-20T06:46:59,246 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:46:59,280 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1351): Waiting on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:59,348 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:46:59,421 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T06:46:59,421 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T06:46:59,481 DEBUG [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1351): Waiting on 03e86be360b2cee8e4aca8882a8efd3d
2024-11-20T06:46:59,499 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/a5e9bc54284645bf89143893d862cc6e
2024-11-20T06:46:59,512 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/.tmp/info/a5e9bc54284645bf89143893d862cc6e as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/a5e9bc54284645bf89143893d862cc6e
2024-11-20T06:46:59,522 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/a5e9bc54284645bf89143893d862cc6e, entries=3, sequenceid=48, filesize=8.0 K
2024-11-20T06:46:59,524 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 444ms, sequenceid=48, compaction requested=true
2024-11-20T06:46:59,524 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a] to archive
2024-11-20T06:46:59,527 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:46:59,530 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/d12a8940c2034715a0ad13d965df975e
2024-11-20T06:46:59,533 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5 to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/ac58005a6e254c09a2987d3ccf2ac8f5
2024-11-20T06:46:59,534 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/info/dc70827c387449649526965472d1483a
2024-11-20T06:46:59,546 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=61c2066a8a0f:42589 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-20T06:46:59,547 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d12a8940c2034715a0ad13d965df975e=12509, ac58005a6e254c09a2987d3ccf2ac8f5=12509, dc70827c387449649526965472d1483a=12509]
2024-11-20T06:46:59,554 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/data/default/TestLogRolling-testSlowSyncLogRolling/03e86be360b2cee8e4aca8882a8efd3d/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-20T06:46:59,554 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:59,555 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03e86be360b2cee8e4aca8882a8efd3d:
Waiting for close lock at 1732085219079Running coprocessor pre-close hooks at 1732085219079Disabling compacts and flushes for region at 1732085219079Disabling writes for close at 1732085219079Obtaining lock to block concurrent updates at 1732085219080 (+1 ms)Preparing flush snapshotting stores in 03e86be360b2cee8e4aca8882a8efd3d at 1732085219080Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732085219080Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d. at 1732085219081 (+1 ms)Flushing 03e86be360b2cee8e4aca8882a8efd3d/info: creating writer at 1732085219081Flushing 03e86be360b2cee8e4aca8882a8efd3d/info: appending metadata at 1732085219086 (+5 ms)Flushing 03e86be360b2cee8e4aca8882a8efd3d/info: closing flushed file at 1732085219086Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@113f0954: reopening flushed file at 1732085219511 (+425 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 03e86be360b2cee8e4aca8882a8efd3d in 444ms, sequenceid=48, compaction requested=true at 1732085219524 (+13 ms)Writing region close event to WAL at 1732085219548 (+24 ms)Running coprocessor post-close hooks at 1732085219554 (+6 ms)Closed at 1732085219554
2024-11-20T06:46:59,555 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732085126329.03e86be360b2cee8e4aca8882a8efd3d.
2024-11-20T06:46:59,681 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,46743,1732085123822; all regions closed.
2024-11-20T06:46:59,683 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,684 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,684 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,684 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,684 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741834_1010 (size=3066)
2024-11-20T06:46:59,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741834_1010 (size=3066)
2024-11-20T06:46:59,694 DEBUG [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs
2024-11-20T06:46:59,695 INFO  [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C46743%2C1732085123822.meta:.meta(num 1732085125871)
2024-11-20T06:46:59,696 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,696 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,696 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,696 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,696 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741847_1023 (size=12695)
2024-11-20T06:46:59,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741847_1023 (size=12695)
2024-11-20T06:46:59,704 DEBUG [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/oldWALs
2024-11-20T06:46:59,705 INFO  [RS:0;61c2066a8a0f:46743 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C46743%2C1732085123822:(num 1732085198940)
2024-11-20T06:46:59,705 DEBUG [RS:0;61c2066a8a0f:46743 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:46:59,705 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:46:59,705 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:46:59,705 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T06:46:59,705 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:46:59,705 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:46:59,706 INFO  [RS:0;61c2066a8a0f:46743 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46743
2024-11-20T06:46:59,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,46743,1732085123822
2024-11-20T06:46:59,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:46:59,742 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:46:59,751 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,46743,1732085123822]
2024-11-20T06:46:59,759 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,46743,1732085123822 already deleted, retry=false
2024-11-20T06:46:59,759 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,46743,1732085123822 expired; onlineServers=0
2024-11-20T06:46:59,759 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,42589,1732085122817' *****
2024-11-20T06:46:59,760 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:46:59,760 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:46:59,760 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:46:59,760 DEBUG [M:0;61c2066a8a0f:42589 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:46:59,760 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:46:59,760 DEBUG [M:0;61c2066a8a0f:42589 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:46:59,760 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085125111 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085125111,5,FailOnTimeoutGroup]
2024-11-20T06:46:59,760 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085125104 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085125104,5,FailOnTimeoutGroup]
2024-11-20T06:46:59,761 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:46:59,761 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:46:59,761 DEBUG [M:0;61c2066a8a0f:42589 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:46:59,761 INFO  [M:0;61c2066a8a0f:42589 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:46:59,761 INFO  [M:0;61c2066a8a0f:42589 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:46:59,762 INFO  [M:0;61c2066a8a0f:42589 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:46:59,762 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:46:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:46:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:46:59,767 DEBUG [M:0;61c2066a8a0f:42589 {}] zookeeper.ZKUtil(347): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:46:59,767 WARN  [M:0;61c2066a8a0f:42589 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:46:59,768 INFO  [M:0;61c2066a8a0f:42589 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/.lastflushedseqids
2024-11-20T06:46:59,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741854_1030 (size=130)
2024-11-20T06:46:59,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741854_1030 (size=130)
2024-11-20T06:46:59,783 INFO  [M:0;61c2066a8a0f:42589 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:46:59,784 INFO  [M:0;61c2066a8a0f:42589 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:46:59,784 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:46:59,784 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:46:59,784 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:46:59,784 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:46:59,784 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:46:59,784 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.01 KB heapSize=29.18 KB
2024-11-20T06:46:59,804 DEBUG [M:0;61c2066a8a0f:42589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a83a14031814faba7f78de5e7f27fd2 is 82, key is hbase:meta,,1/info:regioninfo/1732085125946/Put/seqid=0
2024-11-20T06:46:59,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741855_1031 (size=5672)
2024-11-20T06:46:59,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741855_1031 (size=5672)
2024-11-20T06:46:59,812 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a83a14031814faba7f78de5e7f27fd2
2024-11-20T06:46:59,834 DEBUG [M:0;61c2066a8a0f:42589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/584406f3a35d4cada50699601274841c is 765, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085126852/Put/seqid=0
2024-11-20T06:46:59,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741856_1032 (size=6246)
2024-11-20T06:46:59,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741856_1032 (size=6246)
2024-11-20T06:46:59,842 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.41 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/584406f3a35d4cada50699601274841c
2024-11-20T06:46:59,850 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 584406f3a35d4cada50699601274841c
2024-11-20T06:46:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:46:59,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46743-0x1015847162a0001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:46:59,852 INFO  [RS:0;61c2066a8a0f:46743 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:46:59,852 INFO  [RS:0;61c2066a8a0f:46743 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,46743,1732085123822; zookeeper connection closed.
2024-11-20T06:46:59,852 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e3c27dc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e3c27dc
2024-11-20T06:46:59,853 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:46:59,867 DEBUG [M:0;61c2066a8a0f:42589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b831be0c6f2485fbc297080ccbc0ef5 is 69, key is 61c2066a8a0f,46743,1732085123822/rs:state/1732085125255/Put/seqid=0
2024-11-20T06:46:59,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741857_1033 (size=5156)
2024-11-20T06:46:59,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741857_1033 (size=5156)
2024-11-20T06:46:59,874 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b831be0c6f2485fbc297080ccbc0ef5
2024-11-20T06:46:59,902 DEBUG [M:0;61c2066a8a0f:42589 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3721ff4d16aa471ca4f509c1dda1d7b6 is 52, key is load_balancer_on/state:d/1732085126308/Put/seqid=0
2024-11-20T06:46:59,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741858_1034 (size=5056)
2024-11-20T06:46:59,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741858_1034 (size=5056)
2024-11-20T06:46:59,910 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3721ff4d16aa471ca4f509c1dda1d7b6
2024-11-20T06:46:59,920 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4a83a14031814faba7f78de5e7f27fd2 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4a83a14031814faba7f78de5e7f27fd2
2024-11-20T06:46:59,928 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4a83a14031814faba7f78de5e7f27fd2, entries=8, sequenceid=59, filesize=5.5 K
2024-11-20T06:46:59,929 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/584406f3a35d4cada50699601274841c as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/584406f3a35d4cada50699601274841c
2024-11-20T06:46:59,938 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 584406f3a35d4cada50699601274841c
2024-11-20T06:46:59,938 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/584406f3a35d4cada50699601274841c, entries=6, sequenceid=59, filesize=6.1 K
2024-11-20T06:46:59,940 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3b831be0c6f2485fbc297080ccbc0ef5 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b831be0c6f2485fbc297080ccbc0ef5
2024-11-20T06:46:59,948 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3b831be0c6f2485fbc297080ccbc0ef5, entries=1, sequenceid=59, filesize=5.0 K
2024-11-20T06:46:59,949 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3721ff4d16aa471ca4f509c1dda1d7b6 as hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3721ff4d16aa471ca4f509c1dda1d7b6
2024-11-20T06:46:59,956 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3721ff4d16aa471ca4f509c1dda1d7b6, entries=1, sequenceid=59, filesize=4.9 K
2024-11-20T06:46:59,958 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 173ms, sequenceid=59, compaction requested=false
2024-11-20T06:46:59,964 INFO  [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:46:59,964 DEBUG [M:0;61c2066a8a0f:42589 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085219784Disabling compacts and flushes for region at 1732085219784Disabling writes for close at 1732085219784Obtaining lock to block concurrent updates at 1732085219784Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085219784Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23564, getHeapSize=29816, getOffHeapSize=0, getCellsCount=70 at 1732085219785 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085219786 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085219786Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085219803 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085219803Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085219818 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085219833 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085219833Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085219850 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085219866 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085219866Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085219883 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085219901 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085219901Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20c69d9d: reopening flushed file at 1732085219918 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@621323ee: reopening flushed file at 1732085219928 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59c84d3a: reopening flushed file at 1732085219938 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7baf0fad: reopening flushed file at 1732085219948 (+10 ms)Finished flush of dataSize ~23.01 KB/23564, heapSize ~29.12 KB/29816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 173ms, sequenceid=59, compaction requested=false at 1732085219958 (+10 ms)Writing region close event to WAL at 1732085219963 (+5 ms)Closed at 1732085219963
2024-11-20T06:46:59,965 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,965 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,965 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,965 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,965 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:46:59,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46455 is added to blk_1073741830_1006 (size=27961)
2024-11-20T06:46:59,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39941 is added to blk_1073741830_1006 (size=27961)
2024-11-20T06:46:59,968 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:46:59,968 INFO  [M:0;61c2066a8a0f:42589 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:46:59,969 INFO  [M:0;61c2066a8a0f:42589 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42589
2024-11-20T06:46:59,969 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:00,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:00,084 INFO  [M:0;61c2066a8a0f:42589 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:00,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42589-0x1015847162a0000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:00,089 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58bedcbd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:00,091 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74177ecc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:00,092 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:00,092 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ba9a1a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:00,092 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:00,095 WARN  [BP-617223157-172.17.0.2-1732085118654 heartbeating to localhost/127.0.0.1:33799 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:00,095 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:00,095 WARN  [BP-617223157-172.17.0.2-1732085118654 heartbeating to localhost/127.0.0.1:33799 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-617223157-172.17.0.2-1732085118654 (Datanode Uuid 6f0f480b-2572-42b9-8f90-53e7b50d007a) service to localhost/127.0.0.1:33799
2024-11-20T06:47:00,095 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:00,097 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data3/current/BP-617223157-172.17.0.2-1732085118654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:00,097 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data4/current/BP-617223157-172.17.0.2-1732085118654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:00,098 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:00,100 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27334b48{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:00,100 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@502d300e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:00,100 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:00,100 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e9a9276{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:00,100 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:00,102 WARN  [BP-617223157-172.17.0.2-1732085118654 heartbeating to localhost/127.0.0.1:33799 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:00,102 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:00,102 WARN  [BP-617223157-172.17.0.2-1732085118654 heartbeating to localhost/127.0.0.1:33799 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-617223157-172.17.0.2-1732085118654 (Datanode Uuid a48bc62a-8a04-4fd2-818a-29ee9c147fc2) service to localhost/127.0.0.1:33799
2024-11-20T06:47:00,102 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:00,103 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data1/current/BP-617223157-172.17.0.2-1732085118654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:00,103 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/cluster_dab830e7-6155-5d6d-dff2-f396d793ff85/data/data2/current/BP-617223157-172.17.0.2-1732085118654 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:00,104 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:00,114 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b31d0b7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:00,114 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4fbd44e6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:00,114 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:00,115 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22876862{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:00,115 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64b8518a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:00,126 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:47:00,172 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:47:00,183 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:33799 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/61c2066a8a0f:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33799
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33799
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@69147b9
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: master/61c2066a8a0f:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:33799
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:33799 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:33799 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33799
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33799
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/61c2066a8a0f:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 349), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=5000 (was 5317)
2024-11-20T06:47:00,194 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=232, ProcessCount=11, AvailableMemoryMB=4997
2024-11-20T06:47:00,194 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.log.dir so I do NOT create it in target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/35c1e517-5573-8004-714d-28995eb76425/hadoop.tmp.dir so I do NOT create it in target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009, deleteOnExit=true
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/test.cache.data in system properties and HBase conf
2024-11-20T06:47:00,195 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:47:00,196 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:00,196 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:47:00,197 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:47:00,213 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:00,503 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:00,514 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:00,515 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:00,515 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:00,515 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:00,516 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:00,516 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@641eaf99{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:00,517 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a15ed6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:00,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ce0de36{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/java.io.tmpdir/jetty-localhost-44899-hadoop-hdfs-3_4_1-tests_jar-_-any-16052255196592827551/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:00,646 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3614f662{HTTP/1.1, (http/1.1)}{localhost:44899}
2024-11-20T06:47:00,646 INFO  [Time-limited test {}] server.Server(415): Started @103988ms
2024-11-20T06:47:00,662 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:00,874 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:00,877 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:00,878 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:00,878 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:00,878 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:47:00,879 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa9c156{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:00,879 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e7873b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:00,986 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2cd60cfb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/java.io.tmpdir/jetty-localhost-39445-hadoop-hdfs-3_4_1-tests_jar-_-any-15206196834228762696/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:00,986 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f88e14b{HTTP/1.1, (http/1.1)}{localhost:39445}
2024-11-20T06:47:00,986 INFO  [Time-limited test {}] server.Server(415): Started @104328ms
2024-11-20T06:47:00,988 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:01,022 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:01,026 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:01,027 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:01,027 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:01,027 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:01,028 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d5e070a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:01,028 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@573af0f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:01,145 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2b5e52bc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/java.io.tmpdir/jetty-localhost-44333-hadoop-hdfs-3_4_1-tests_jar-_-any-1448196449793517261/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:01,146 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@882842c{HTTP/1.1, (http/1.1)}{localhost:44333}
2024-11-20T06:47:01,146 INFO  [Time-limited test {}] server.Server(415): Started @104488ms
2024-11-20T06:47:01,147 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:01,734 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data1/current/BP-303753975-172.17.0.2-1732085220225/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:01,734 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data2/current/BP-303753975-172.17.0.2-1732085220225/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:01,753 WARN  [Thread-418 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:01,756 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8de9c3154148de9e with lease ID 0x43dd40695165a544: Processing first storage report for DS-cb8ececf-96f7-4a15-b755-fdafefe4851c from datanode DatanodeRegistration(127.0.0.1:39059, datanodeUuid=7fb2cd8a-d55a-429b-80ac-c5fb890b5d45, infoPort=39725, infoSecurePort=0, ipcPort=37217, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225)
2024-11-20T06:47:01,757 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8de9c3154148de9e with lease ID 0x43dd40695165a544: from storage DS-cb8ececf-96f7-4a15-b755-fdafefe4851c node DatanodeRegistration(127.0.0.1:39059, datanodeUuid=7fb2cd8a-d55a-429b-80ac-c5fb890b5d45, infoPort=39725, infoSecurePort=0, ipcPort=37217, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:01,757 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8de9c3154148de9e with lease ID 0x43dd40695165a544: Processing first storage report for DS-878a4c19-8e91-4999-93de-13f617a172a1 from datanode DatanodeRegistration(127.0.0.1:39059, datanodeUuid=7fb2cd8a-d55a-429b-80ac-c5fb890b5d45, infoPort=39725, infoSecurePort=0, ipcPort=37217, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225)
2024-11-20T06:47:01,757 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8de9c3154148de9e with lease ID 0x43dd40695165a544: from storage DS-878a4c19-8e91-4999-93de-13f617a172a1 node DatanodeRegistration(127.0.0.1:39059, datanodeUuid=7fb2cd8a-d55a-429b-80ac-c5fb890b5d45, infoPort=39725, infoSecurePort=0, ipcPort=37217, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:01,915 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data3/current/BP-303753975-172.17.0.2-1732085220225/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:01,915 WARN  [Thread-466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data4/current/BP-303753975-172.17.0.2-1732085220225/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:01,938 WARN  [Thread-441 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:01,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba30a874702ac4de with lease ID 0x43dd40695165a545: Processing first storage report for DS-c1337f8f-9c80-4ba8-bba8-1e3f8ee1ff13 from datanode DatanodeRegistration(127.0.0.1:41297, datanodeUuid=ff0ef066-eb68-4199-b649-f9f0b93f1857, infoPort=39297, infoSecurePort=0, ipcPort=34049, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225)
2024-11-20T06:47:01,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba30a874702ac4de with lease ID 0x43dd40695165a545: from storage DS-c1337f8f-9c80-4ba8-bba8-1e3f8ee1ff13 node DatanodeRegistration(127.0.0.1:41297, datanodeUuid=ff0ef066-eb68-4199-b649-f9f0b93f1857, infoPort=39297, infoSecurePort=0, ipcPort=34049, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:01,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xba30a874702ac4de with lease ID 0x43dd40695165a545: Processing first storage report for DS-b6c4901a-f7a2-4d28-a041-52b790401466 from datanode DatanodeRegistration(127.0.0.1:41297, datanodeUuid=ff0ef066-eb68-4199-b649-f9f0b93f1857, infoPort=39297, infoSecurePort=0, ipcPort=34049, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225)
2024-11-20T06:47:01,941 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xba30a874702ac4de with lease ID 0x43dd40695165a545: from storage DS-b6c4901a-f7a2-4d28-a041-52b790401466 node DatanodeRegistration(127.0.0.1:41297, datanodeUuid=ff0ef066-eb68-4199-b649-f9f0b93f1857, infoPort=39297, infoSecurePort=0, ipcPort=34049, storageInfo=lv=-57;cid=testClusterID;nsid=1218343520;c=1732085220225), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:47:01,997 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2
2024-11-20T06:47:02,002 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/zookeeper_0, clientPort=62772, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:47:02,003 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62772
2024-11-20T06:47:02,003 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,005 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,017 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:02,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:02,020 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467 with version=8
2024-11-20T06:47:02,020 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:47:02,022 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:02,022 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:47:02,023 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:02,024 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34413
2024-11-20T06:47:02,026 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34413 connecting to ZooKeeper ensemble=127.0.0.1:62772
2024-11-20T06:47:02,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344130x0, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:02,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34413-0x10158489d350000 connected
2024-11-20T06:47:02,156 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,158 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:02,162 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467, hbase.cluster.distributed=false
2024-11-20T06:47:02,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:02,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34413
2024-11-20T06:47:02,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34413
2024-11-20T06:47:02,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34413
2024-11-20T06:47:02,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34413
2024-11-20T06:47:02,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34413
2024-11-20T06:47:02,189 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:02,189 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,189 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,190 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:02,190 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:02,190 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:02,190 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:47:02,190 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:02,191 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39557
2024-11-20T06:47:02,193 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39557 connecting to ZooKeeper ensemble=127.0.0.1:62772
2024-11-20T06:47:02,194 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,196 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395570x0, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:02,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39557-0x10158489d350001 connected
2024-11-20T06:47:02,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:02,212 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:47:02,214 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:47:02,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:47:02,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:02,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39557
2024-11-20T06:47:02,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39557
2024-11-20T06:47:02,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39557
2024-11-20T06:47:02,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39557
2024-11-20T06:47:02,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39557
2024-11-20T06:47:02,237 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:34413
2024-11-20T06:47:02,238 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:02,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:02,249 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:47:02,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,256 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:47:02,257 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,34413,1732085222022 from backup master directory
2024-11-20T06:47:02,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:02,264 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:47:02,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:02,264 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,272 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/hbase.id] with ID: b00d5755-39b5-450d-bc11-5b9e27c2a123
2024-11-20T06:47:02,272 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/.tmp/hbase.id
2024-11-20T06:47:02,285 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:47:02,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:47:02,286 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/.tmp/hbase.id]:[hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/hbase.id]
2024-11-20T06:47:02,303 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:02,303 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:47:02,306 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-20T06:47:02,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:47:02,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:47:02,324 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:47:02,325 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:47:02,325 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:02,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:47:02,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:47:02,337 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store
2024-11-20T06:47:02,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:47:02,349 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:47:02,350 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:02,350 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:47:02,350 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:02,350 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:02,351 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:47:02,351 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:02,351 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:02,351 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085222350Disabling compacts and flushes for region at 1732085222350Disabling writes for close at 1732085222351 (+1 ms)Writing region close event to WAL at 1732085222351Closed at 1732085222351
2024-11-20T06:47:02,352 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/.initializing
2024-11-20T06:47:02,353 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/WALs/61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,356 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C34413%2C1732085222022, suffix=, logDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/WALs/61c2066a8a0f,34413,1732085222022, archiveDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/oldWALs, maxLogs=10
2024-11-20T06:47:02,356 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C34413%2C1732085222022.1732085222356
2024-11-20T06:47:02,368 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/WALs/61c2066a8a0f,34413,1732085222022/61c2066a8a0f%2C34413%2C1732085222022.1732085222356
2024-11-20T06:47:02,375 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39725:39725),(127.0.0.1/127.0.0.1:39297:39297)]
2024-11-20T06:47:02,376 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:47:02,376 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:02,376 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,376 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,379 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:02,381 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:47:02,381 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,382 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:02,382 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:02,383 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:47:02,383 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,384 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:02,384 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:02,386 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:47:02,387 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,388 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:02,388 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:02,390 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:47:02,390 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:02,391 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,392 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,392 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,394 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,394 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,394 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:47:02,396 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:02,398 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:47:02,399 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=815707, jitterRate=0.03722585737705231}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:47:02,400 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085222376Initializing all the Stores at 1732085222377 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085222378 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085222378Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085222378Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085222378Cleaning up temporary data from old regions at 1732085222394 (+16 ms)Region opened successfully at 1732085222400 (+6 ms)
2024-11-20T06:47:02,402 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:47:02,407 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db69783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:47:02,408 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:47:02,408 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:47:02,408 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:47:02,409 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:47:02,409 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:47:02,410 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:47:02,410 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:47:02,413 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:47:02,414 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:47:02,444 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:47:02,445 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:47:02,446 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:47:02,456 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:47:02,456 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:47:02,457 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:47:02,464 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:47:02,465 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:47:02,472 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:47:02,474 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:47:02,486 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:47:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:02,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,498 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,34413,1732085222022, sessionid=0x10158489d350000, setting cluster-up flag (Was=false)
2024-11-20T06:47:02,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,539 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:47:02,541 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:02,594 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:47:02,596 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:02,597 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:47:02,599 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:02,599 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:47:02,600 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:47:02,600 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,34413,1732085222022
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,604 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:47:02,605 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,607 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:02,607 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:47:02,607 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085252607
2024-11-20T06:47:02,607 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,608 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:47:02,608 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:47:02,608 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:47:02,609 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085222609,5,FailOnTimeoutGroup]
2024-11-20T06:47:02,609 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085222609,5,FailOnTimeoutGroup]
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,609 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,616 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:47:02,617 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:47:02,618 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:47:02,618 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467
2024-11-20T06:47:02,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:47:02,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:47:02,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:02,635 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:47:02,636 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(746): ClusterId : b00d5755-39b5-450d-bc11-5b9e27c2a123
2024-11-20T06:47:02,636 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:47:02,637 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:47:02,637 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,638 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:02,638 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:47:02,640 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:47:02,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:02,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:47:02,643 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:47:02,643 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:02,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:47:02,645 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:47:02,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:02,646 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:02,646 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:47:02,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740
2024-11-20T06:47:02,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740
2024-11-20T06:47:02,648 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:47:02,649 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:47:02,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:47:02,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:47:02,650 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:47:02,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:47:02,654 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:47:02,655 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=759031, jitterRate=-0.03484314680099487}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:47:02,656 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:47:02,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085222629Initializing all the Stores at 1732085222630 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085222630Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085222635 (+5 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085222635Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085222635Cleaning up temporary data from old regions at 1732085222650 (+15 ms)Region opened successfully at 1732085222656 (+6 ms)
2024-11-20T06:47:02,656 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:47:02,657 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:47:02,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:47:02,657 DEBUG [RS:0;61c2066a8a0f:39557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6557fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:47:02,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:47:02,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:47:02,659 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:02,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085222656Disabling compacts and flushes for region at 1732085222656Disabling writes for close at 1732085222657 (+1 ms)Writing region close event to WAL at 1732085222659 (+2 ms)Closed at 1732085222659
2024-11-20T06:47:02,660 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:02,660 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:47:02,661 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:47:02,663 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:47:02,665 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:47:02,671 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:39557
2024-11-20T06:47:02,671 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:47:02,671 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:47:02,671 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:47:02,673 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,34413,1732085222022 with port=39557, startcode=1732085222189
2024-11-20T06:47:02,673 DEBUG [RS:0;61c2066a8a0f:39557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:47:02,677 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:47:02,678 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,679 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34413 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,681 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467
2024-11-20T06:47:02,681 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42115
2024-11-20T06:47:02,681 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:47:02,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:02,690 DEBUG [RS:0;61c2066a8a0f:39557 {}] zookeeper.ZKUtil(111): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,690 WARN  [RS:0;61c2066a8a0f:39557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:47:02,690 INFO  [RS:0;61c2066a8a0f:39557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:02,690 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/WALs/61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,690 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,39557,1732085222189]
2024-11-20T06:47:02,696 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:47:02,699 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:47:02,700 INFO  [RS:0;61c2066a8a0f:39557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:47:02,700 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,700 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:47:02,701 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:47:02,701 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,701 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,701 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:02,702 DEBUG [RS:0;61c2066a8a0f:39557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,704 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,39557,1732085222189-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:47:02,721 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:47:02,721 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,39557,1732085222189-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,721 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,721 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.Replication(171): 61c2066a8a0f,39557,1732085222189 started
2024-11-20T06:47:02,738 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:02,738 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,39557,1732085222189, RpcServer on 61c2066a8a0f/172.17.0.2:39557, sessionid=0x10158489d350001
2024-11-20T06:47:02,738 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:47:02,738 DEBUG [RS:0;61c2066a8a0f:39557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,739 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,39557,1732085222189'
2024-11-20T06:47:02,739 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:47:02,739 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:47:02,740 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:47:02,740 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:47:02,740 DEBUG [RS:0;61c2066a8a0f:39557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:02,740 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,39557,1732085222189'
2024-11-20T06:47:02,740 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:47:02,741 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:47:02,741 DEBUG [RS:0;61c2066a8a0f:39557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:47:02,741 INFO  [RS:0;61c2066a8a0f:39557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:47:02,741 INFO  [RS:0;61c2066a8a0f:39557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:47:02,816 WARN  [61c2066a8a0f:34413 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:47:02,844 INFO  [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C39557%2C1732085222189, suffix=, logDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/WALs/61c2066a8a0f,39557,1732085222189, archiveDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/oldWALs, maxLogs=32
2024-11-20T06:47:02,847 INFO  [RS:0;61c2066a8a0f:39557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C39557%2C1732085222189.1732085222847
2024-11-20T06:47:02,857 INFO  [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/WALs/61c2066a8a0f,39557,1732085222189/61c2066a8a0f%2C39557%2C1732085222189.1732085222847
2024-11-20T06:47:02,858 DEBUG [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39297:39297),(127.0.0.1/127.0.0.1:39725:39725)]
2024-11-20T06:47:03,066 DEBUG [61c2066a8a0f:34413 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:47:03,067 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:03,069 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,39557,1732085222189, state=OPENING
2024-11-20T06:47:03,107 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:47:03,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:03,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:03,157 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:47:03,157 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,39557,1732085222189}]
2024-11-20T06:47:03,157 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:03,158 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:03,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:47:03,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:47:03,219 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-20T06:47:03,313 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:47:03,316 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38981, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:47:03,323 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:47:03,323 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:03,325 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C39557%2C1732085222189.meta, suffix=.meta, logDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/WALs/61c2066a8a0f,39557,1732085222189, archiveDir=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/oldWALs, maxLogs=32
2024-11-20T06:47:03,327 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C39557%2C1732085222189.meta.1732085223327.meta
2024-11-20T06:47:03,333 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/WALs/61c2066a8a0f,39557,1732085222189/61c2066a8a0f%2C39557%2C1732085222189.meta.1732085223327.meta
2024-11-20T06:47:03,334 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39725:39725),(127.0.0.1/127.0.0.1:39297:39297)]
2024-11-20T06:47:03,335 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:47:03,335 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:47:03,336 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:47:03,336 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:47:03,336 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:47:03,336 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:03,336 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:47:03,336 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:47:03,338 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:47:03,339 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:47:03,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:03,340 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:03,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:47:03,342 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:47:03,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:03,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:03,343 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:47:03,344 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:47:03,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:03,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:03,345 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:47:03,347 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:47:03,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:03,347 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:03,347 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:47:03,348 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740
2024-11-20T06:47:03,350 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740
2024-11-20T06:47:03,352 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:47:03,352 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:47:03,353 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:47:03,355 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:47:03,356 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717003, jitterRate=-0.08828380703926086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:47:03,356 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:47:03,357 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085223336Writing region info on filesystem at 1732085223336Initializing all the Stores at 1732085223337 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085223337Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085223338 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085223338Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085223338Cleaning up temporary data from old regions at 1732085223352 (+14 ms)Running coprocessor post-open hooks at 1732085223356 (+4 ms)Region opened successfully at 1732085223357 (+1 ms)
2024-11-20T06:47:03,358 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085223312
2024-11-20T06:47:03,361 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:47:03,361 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:47:03,362 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:03,364 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,39557,1732085222189, state=OPEN
2024-11-20T06:47:03,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:47:03,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:47:03,408 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:03,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:03,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:03,414 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:47:03,414 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,39557,1732085222189 in 252 msec
2024-11-20T06:47:03,419 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:47:03,419 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec
2024-11-20T06:47:03,420 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:03,420 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:47:03,422 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:47:03,422 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,39557,1732085222189, seqNum=-1]
2024-11-20T06:47:03,422 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:47:03,424 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60001, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:47:03,433 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 832 msec
2024-11-20T06:47:03,433 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085223433, completionTime=-1
2024-11-20T06:47:03,433 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:47:03,433 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:47:03,436 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:47:03,436 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085283436
2024-11-20T06:47:03,436 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085343436
2024-11-20T06:47:03,436 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-20T06:47:03,436 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,437 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,437 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,437 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:34413, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,437 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,437 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,439 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:47:03,442 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.178sec
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:47:03,443 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:47:03,446 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:47:03,446 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:47:03,446 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34413,1732085222022-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:03,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aadf114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:03,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,34413,-1 for getting cluster id
2024-11-20T06:47:03,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:47:03,543 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b00d5755-39b5-450d-bc11-5b9e27c2a123'
2024-11-20T06:47:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:47:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b00d5755-39b5-450d-bc11-5b9e27c2a123"

2024-11-20T06:47:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa3ce73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:03,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,34413,-1]
2024-11-20T06:47:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:47:03,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:03,549 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:47:03,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510cc70d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:03,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:47:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,39557,1732085222189, seqNum=-1]
2024-11-20T06:47:03,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:47:03,555 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:47:03,557 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:03,558 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:03,561 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:47:03,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:47:03,561 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:47:03,561 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:03,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:03,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:03,561 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:47:03,561 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:47:03,562 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2014902235, stopped=false
2024-11-20T06:47:03,562 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,34413,1732085222022
2024-11-20T06:47:03,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:03,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:03,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:03,586 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:47:03,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:03,586 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:47:03,586 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:03,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:03,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:03,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:03,587 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,39557,1732085222189' *****
2024-11-20T06:47:03,587 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:47:03,587 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:47:03,587 INFO  [RS:0;61c2066a8a0f:39557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:47:03,587 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:47:03,587 INFO  [RS:0;61c2066a8a0f:39557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:47:03,587 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:03,587 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:39557.
2024-11-20T06:47:03,588 DEBUG [RS:0;61c2066a8a0f:39557 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:03,588 DEBUG [RS:0;61c2066a8a0f:39557 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:47:03,588 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T06:47:03,588 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:47:03,588 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:47:03,588 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T06:47:03,588 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:47:03,589 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:47:03,589 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:47:03,589 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:47:03,589 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-20T06:47:03,614 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/.tmp/ns/d721d8917cb54e56bb3bbe6b9c9b2c74 is 43, key is default/ns:d/1732085223425/Put/seqid=0
2024-11-20T06:47:03,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741835_1011 (size=5153)
2024-11-20T06:47:03,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741835_1011 (size=5153)
2024-11-20T06:47:03,729 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T06:47:03,729 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T06:47:03,789 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T06:47:03,989 DEBUG [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T06:47:04,021 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/.tmp/ns/d721d8917cb54e56bb3bbe6b9c9b2c74
2024-11-20T06:47:04,034 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/.tmp/ns/d721d8917cb54e56bb3bbe6b9c9b2c74 as hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/ns/d721d8917cb54e56bb3bbe6b9c9b2c74
2024-11-20T06:47:04,043 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/ns/d721d8917cb54e56bb3bbe6b9c9b2c74, entries=2, sequenceid=6, filesize=5.0 K
2024-11-20T06:47:04,044 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 455ms, sequenceid=6, compaction requested=false
2024-11-20T06:47:04,044 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:47:04,050 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-20T06:47:04,051 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:47:04,051 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:04,051 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085223588Running coprocessor pre-close hooks at 1732085223588Disabling compacts and flushes for region at 1732085223588Disabling writes for close at 1732085223589 (+1 ms)Obtaining lock to block concurrent updates at 1732085223589Preparing flush snapshotting stores in 1588230740 at 1732085223589Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732085223589Flushing stores of hbase:meta,,1.1588230740 at 1732085223590 (+1 ms)Flushing 1588230740/ns: creating writer at 1732085223590Flushing 1588230740/ns: appending metadata at 1732085223613 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1732085223613Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16ccea0: reopening flushed file at 1732085224033 (+420 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 455ms, sequenceid=6, compaction requested=false at 1732085224044 (+11 ms)Writing region close event to WAL at 1732085224046 (+2 ms)Running coprocessor post-close hooks at 1732085224051 (+5 ms)Closed at 1732085224051
2024-11-20T06:47:04,052 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:04,189 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,39557,1732085222189; all regions closed.
2024-11-20T06:47:04,190 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,190 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,191 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,191 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,191 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741834_1010 (size=1152)
2024-11-20T06:47:04,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741834_1010 (size=1152)
2024-11-20T06:47:04,200 DEBUG [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/oldWALs
2024-11-20T06:47:04,200 INFO  [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C39557%2C1732085222189.meta:.meta(num 1732085223327)
2024-11-20T06:47:04,200 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,201 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,201 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,201 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,201 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,203 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741833_1009 (size=93)
2024-11-20T06:47:04,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741833_1009 (size=93)
2024-11-20T06:47:04,206 DEBUG [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/oldWALs
2024-11-20T06:47:04,206 INFO  [RS:0;61c2066a8a0f:39557 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C39557%2C1732085222189:(num 1732085222847)
2024-11-20T06:47:04,206 DEBUG [RS:0;61c2066a8a0f:39557 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:04,207 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:04,207 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:47:04,207 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T06:47:04,207 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:47:04,207 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:47:04,207 INFO  [RS:0;61c2066a8a0f:39557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39557
2024-11-20T06:47:04,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:04,256 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:04,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,39557,1732085222189
2024-11-20T06:47:04,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:04,301 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:04,302 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,39557,1732085222189]
2024-11-20T06:47:04,319 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,39557,1732085222189 already deleted, retry=false
2024-11-20T06:47:04,319 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,39557,1732085222189 expired; onlineServers=0
2024-11-20T06:47:04,319 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,34413,1732085222022' *****
2024-11-20T06:47:04,319 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:47:04,319 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:47:04,319 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:47:04,320 DEBUG [M:0;61c2066a8a0f:34413 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:47:04,320 DEBUG [M:0;61c2066a8a0f:34413 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:47:04,320 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085222609 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085222609,5,FailOnTimeoutGroup]
2024-11-20T06:47:04,320 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:47:04,320 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085222609 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085222609,5,FailOnTimeoutGroup]
2024-11-20T06:47:04,320 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:47:04,320 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:47:04,320 DEBUG [M:0;61c2066a8a0f:34413 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:47:04,320 INFO  [M:0;61c2066a8a0f:34413 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:47:04,321 INFO  [M:0;61c2066a8a0f:34413 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:47:04,321 INFO  [M:0;61c2066a8a0f:34413 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:47:04,321 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:47:04,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:47:04,330 DEBUG [M:0;61c2066a8a0f:34413 {}] zookeeper.ZKUtil(347): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:47:04,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:04,330 WARN  [M:0;61c2066a8a0f:34413 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:47:04,331 INFO  [M:0;61c2066a8a0f:34413 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/.lastflushedseqids
2024-11-20T06:47:04,341 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741836_1012 (size=99)
2024-11-20T06:47:04,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741836_1012 (size=99)
2024-11-20T06:47:04,343 INFO  [M:0;61c2066a8a0f:34413 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:47:04,343 INFO  [M:0;61c2066a8a0f:34413 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:47:04,343 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:47:04,343 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:04,343 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:04,343 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:47:04,343 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:04,344 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-20T06:47:04,366 DEBUG [M:0;61c2066a8a0f:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb4b8948c6334e5fa7b60fc1c98b1d46 is 82, key is hbase:meta,,1/info:regioninfo/1732085223362/Put/seqid=0
2024-11-20T06:47:04,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741837_1013 (size=5672)
2024-11-20T06:47:04,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741837_1013 (size=5672)
2024-11-20T06:47:04,372 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb4b8948c6334e5fa7b60fc1c98b1d46
2024-11-20T06:47:04,402 DEBUG [M:0;61c2066a8a0f:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e637f1f314f4099bd5b19c94bf19099 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732085223432/Put/seqid=0
2024-11-20T06:47:04,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741838_1014 (size=5275)
2024-11-20T06:47:04,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741838_1014 (size=5275)
2024-11-20T06:47:04,409 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e637f1f314f4099bd5b19c94bf19099
2024-11-20T06:47:04,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:04,411 INFO  [RS:0;61c2066a8a0f:39557 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:04,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39557-0x10158489d350001, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:04,411 INFO  [RS:0;61c2066a8a0f:39557 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,39557,1732085222189; zookeeper connection closed.
2024-11-20T06:47:04,411 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7644a2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7644a2
2024-11-20T06:47:04,411 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:47:04,433 DEBUG [M:0;61c2066a8a0f:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6c55218d4034cd8ad73e762dfd73f68 is 69, key is 61c2066a8a0f,39557,1732085222189/rs:state/1732085222679/Put/seqid=0
2024-11-20T06:47:04,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741839_1015 (size=5156)
2024-11-20T06:47:04,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741839_1015 (size=5156)
2024-11-20T06:47:04,439 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6c55218d4034cd8ad73e762dfd73f68
2024-11-20T06:47:04,464 DEBUG [M:0;61c2066a8a0f:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fcbbff6954346508da45b402edc4da1 is 52, key is load_balancer_on/state:d/1732085223559/Put/seqid=0
2024-11-20T06:47:04,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741840_1016 (size=5056)
2024-11-20T06:47:04,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741840_1016 (size=5056)
2024-11-20T06:47:04,705 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:04,772 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:47:04,782 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:04,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:04,872 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fcbbff6954346508da45b402edc4da1
2024-11-20T06:47:04,882 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fb4b8948c6334e5fa7b60fc1c98b1d46 as hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb4b8948c6334e5fa7b60fc1c98b1d46
2024-11-20T06:47:04,890 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fb4b8948c6334e5fa7b60fc1c98b1d46, entries=8, sequenceid=29, filesize=5.5 K
2024-11-20T06:47:04,891 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e637f1f314f4099bd5b19c94bf19099 as hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e637f1f314f4099bd5b19c94bf19099
2024-11-20T06:47:04,899 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e637f1f314f4099bd5b19c94bf19099, entries=3, sequenceid=29, filesize=5.2 K
2024-11-20T06:47:04,901 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6c55218d4034cd8ad73e762dfd73f68 as hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6c55218d4034cd8ad73e762dfd73f68
2024-11-20T06:47:04,910 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6c55218d4034cd8ad73e762dfd73f68, entries=1, sequenceid=29, filesize=5.0 K
2024-11-20T06:47:04,912 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/0fcbbff6954346508da45b402edc4da1 as hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fcbbff6954346508da45b402edc4da1
2024-11-20T06:47:04,919 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42115/user/jenkins/test-data/f2e991f1-356e-2981-4962-e5174b55b467/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/0fcbbff6954346508da45b402edc4da1, entries=1, sequenceid=29, filesize=4.9 K
2024-11-20T06:47:04,920 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 577ms, sequenceid=29, compaction requested=false
2024-11-20T06:47:04,921 INFO  [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:04,922 DEBUG [M:0;61c2066a8a0f:34413 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085224343Disabling compacts and flushes for region at 1732085224343Disabling writes for close at 1732085224343Obtaining lock to block concurrent updates at 1732085224344 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085224344Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732085224344Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085224345 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085224345Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085224365 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085224365Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085224379 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085224401 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085224401Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085224415 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085224432 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085224432Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085224446 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085224464 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085224464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37663f6d: reopening flushed file at 1732085224880 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3494274d: reopening flushed file at 1732085224890 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ee3a0ca: reopening flushed file at 1732085224899 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2e195b: reopening flushed file at 1732085224910 (+11 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 577ms, sequenceid=29, compaction requested=false at 1732085224920 (+10 ms)Writing region close event to WAL at 1732085224921 (+1 ms)Closed at 1732085224921
2024-11-20T06:47:04,923 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,923 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,923 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,923 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,924 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:04,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41297 is added to blk_1073741830_1006 (size=10311)
2024-11-20T06:47:04,926 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39059 is added to blk_1073741830_1006 (size=10311)
2024-11-20T06:47:04,927 INFO  [M:0;61c2066a8a0f:34413 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:47:04,927 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:47:04,927 INFO  [M:0;61c2066a8a0f:34413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34413
2024-11-20T06:47:04,927 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:05,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:05,080 INFO  [M:0;61c2066a8a0f:34413 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:05,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x10158489d350000, quorum=127.0.0.1:62772, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:05,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2b5e52bc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:05,084 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@882842c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:05,085 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:05,085 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@573af0f2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:05,085 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d5e070a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:05,086 WARN  [BP-303753975-172.17.0.2-1732085220225 heartbeating to localhost/127.0.0.1:42115 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:05,086 WARN  [BP-303753975-172.17.0.2-1732085220225 heartbeating to localhost/127.0.0.1:42115 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-303753975-172.17.0.2-1732085220225 (Datanode Uuid ff0ef066-eb68-4199-b649-f9f0b93f1857) service to localhost/127.0.0.1:42115
2024-11-20T06:47:05,086 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:05,086 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:05,087 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data3/current/BP-303753975-172.17.0.2-1732085220225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:05,087 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data4/current/BP-303753975-172.17.0.2-1732085220225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:05,087 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:05,090 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2cd60cfb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:05,090 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f88e14b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:05,090 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:05,091 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e7873b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:05,091 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa9c156{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:05,092 WARN  [BP-303753975-172.17.0.2-1732085220225 heartbeating to localhost/127.0.0.1:42115 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:05,092 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:05,092 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:05,092 WARN  [BP-303753975-172.17.0.2-1732085220225 heartbeating to localhost/127.0.0.1:42115 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-303753975-172.17.0.2-1732085220225 (Datanode Uuid 7fb2cd8a-d55a-429b-80ac-c5fb890b5d45) service to localhost/127.0.0.1:42115
2024-11-20T06:47:05,093 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data1/current/BP-303753975-172.17.0.2-1732085220225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:05,093 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/cluster_b162b519-7a6c-5126-3973-dcd8980d4009/data/data2/current/BP-303753975-172.17.0.2-1732085220225 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:05,093 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:05,100 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ce0de36{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:05,101 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3614f662{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:05,101 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:05,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a15ed6a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:05,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@641eaf99{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:05,107 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:47:05,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:47:05,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:47:05,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.log.dir so I do NOT create it in target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605
2024-11-20T06:47:05,128 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d260ea-8d04-c0eb-7760-b8e1480cace2/hadoop.tmp.dir so I do NOT create it in target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053, deleteOnExit=true
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/test.cache.data in system properties and HBase conf
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:47:05,129 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:47:05,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:05,130 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:47:05,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:47:05,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:47:05,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:05,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:47:05,131 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:47:05,145 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:05,421 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:05,427 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:05,429 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:05,429 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:05,429 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:05,430 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:05,430 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3150e6db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:05,430 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d790455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:05,537 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7982676d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-37797-hadoop-hdfs-3_4_1-tests_jar-_-any-2662198912486842154/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:05,538 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2efbdc75{HTTP/1.1, (http/1.1)}{localhost:37797}
2024-11-20T06:47:05,538 INFO  [Time-limited test {}] server.Server(415): Started @108880ms
2024-11-20T06:47:05,551 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:05,746 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:05,750 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:05,750 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:05,750 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:05,751 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:47:05,755 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf32f74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:05,755 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bb5d847{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:05,867 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5538b075{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-38515-hadoop-hdfs-3_4_1-tests_jar-_-any-1865118718501395273/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:05,868 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f87a993{HTTP/1.1, (http/1.1)}{localhost:38515}
2024-11-20T06:47:05,868 INFO  [Time-limited test {}] server.Server(415): Started @109210ms
2024-11-20T06:47:05,870 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:05,903 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:05,908 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:05,908 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:05,909 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:05,909 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:05,909 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32403ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:05,910 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bb19ef9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:06,016 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@272348fe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-43413-hadoop-hdfs-3_4_1-tests_jar-_-any-3986679666535546181/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:06,017 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10b53169{HTTP/1.1, (http/1.1)}{localhost:43413}
2024-11-20T06:47:06,017 INFO  [Time-limited test {}] server.Server(415): Started @109359ms
2024-11-20T06:47:06,018 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:06,599 WARN  [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data1/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:06,599 WARN  [Thread-675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data2/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:06,627 WARN  [Thread-638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:06,630 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x855e9cb0960fcc0b with lease ID 0xd87bea97461ac232: Processing first storage report for DS-e12a774d-9445-4470-8205-7833e690accd from datanode DatanodeRegistration(127.0.0.1:37307, datanodeUuid=a306182b-ad6d-4234-b341-832e7ffd241e, infoPort=36513, infoSecurePort=0, ipcPort=38657, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:06,630 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x855e9cb0960fcc0b with lease ID 0xd87bea97461ac232: from storage DS-e12a774d-9445-4470-8205-7833e690accd node DatanodeRegistration(127.0.0.1:37307, datanodeUuid=a306182b-ad6d-4234-b341-832e7ffd241e, infoPort=36513, infoSecurePort=0, ipcPort=38657, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:47:06,631 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x855e9cb0960fcc0b with lease ID 0xd87bea97461ac232: Processing first storage report for DS-b56fd509-f26e-4c48-9ba2-5a582c805b1f from datanode DatanodeRegistration(127.0.0.1:37307, datanodeUuid=a306182b-ad6d-4234-b341-832e7ffd241e, infoPort=36513, infoSecurePort=0, ipcPort=38657, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:06,631 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x855e9cb0960fcc0b with lease ID 0xd87bea97461ac232: from storage DS-b56fd509-f26e-4c48-9ba2-5a582c805b1f node DatanodeRegistration(127.0.0.1:37307, datanodeUuid=a306182b-ad6d-4234-b341-832e7ffd241e, infoPort=36513, infoSecurePort=0, ipcPort=38657, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:06,771 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data3/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:06,772 WARN  [Thread-686 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data4/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:06,797 WARN  [Thread-661 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:06,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dc3303bdc7b9be6 with lease ID 0xd87bea97461ac233: Processing first storage report for DS-0356e157-8ecb-4880-a294-8b712f127c1d from datanode DatanodeRegistration(127.0.0.1:39949, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=40221, infoSecurePort=0, ipcPort=32905, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:06,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dc3303bdc7b9be6 with lease ID 0xd87bea97461ac233: from storage DS-0356e157-8ecb-4880-a294-8b712f127c1d node DatanodeRegistration(127.0.0.1:39949, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=40221, infoSecurePort=0, ipcPort=32905, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:06,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dc3303bdc7b9be6 with lease ID 0xd87bea97461ac233: Processing first storage report for DS-245946a7-a5ae-4fe8-89b6-6a61b564def7 from datanode DatanodeRegistration(127.0.0.1:39949, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=40221, infoSecurePort=0, ipcPort=32905, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:06,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dc3303bdc7b9be6 with lease ID 0xd87bea97461ac233: from storage DS-245946a7-a5ae-4fe8-89b6-6a61b564def7 node DatanodeRegistration(127.0.0.1:39949, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=40221, infoSecurePort=0, ipcPort=32905, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:06,853 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605
2024-11-20T06:47:06,856 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/zookeeper_0, clientPort=62382, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:47:06,857 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62382
2024-11-20T06:47:06,857 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:06,859 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:06,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:06,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:06,877 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a with version=8
2024-11-20T06:47:06,877 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:47:06,879 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:47:06,880 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:06,881 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38687
2024-11-20T06:47:06,882 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38687 connecting to ZooKeeper ensemble=127.0.0.1:62382
2024-11-20T06:47:06,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386870x0, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:06,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38687-0x1015848b02d0000 connected
2024-11-20T06:47:07,022 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:07,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:07,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:07,027 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a, hbase.cluster.distributed=false
2024-11-20T06:47:07,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:07,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38687
2024-11-20T06:47:07,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38687
2024-11-20T06:47:07,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38687
2024-11-20T06:47:07,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38687
2024-11-20T06:47:07,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38687
2024-11-20T06:47:07,061 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:47:07,062 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:07,063 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41993
2024-11-20T06:47:07,065 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41993 connecting to ZooKeeper ensemble=127.0.0.1:62382
2024-11-20T06:47:07,066 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:07,069 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:07,105 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419930x0, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:07,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:07,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41993-0x1015848b02d0001 connected
2024-11-20T06:47:07,106 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:47:07,106 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:47:07,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:47:07,108 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:07,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41993
2024-11-20T06:47:07,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41993
2024-11-20T06:47:07,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41993
2024-11-20T06:47:07,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41993
2024-11-20T06:47:07,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41993
2024-11-20T06:47:07,132 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:38687
2024-11-20T06:47:07,133 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,143 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:07,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:07,144 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,155 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:47:07,155 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,155 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:47:07,156 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,38687,1732085226879 from backup master directory
2024-11-20T06:47:07,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,163 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:07,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:07,164 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:47:07,164 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,169 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/hbase.id] with ID: 272846e8-f98b-42e6-8b48-4a58ad6b25d4
2024-11-20T06:47:07,169 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/.tmp/hbase.id
2024-11-20T06:47:07,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:47:07,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:47:07,182 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/.tmp/hbase.id]:[hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/hbase.id]
2024-11-20T06:47:07,197 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:07,197 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:47:07,198 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T06:47:07,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,210 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:47:07,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:47:07,225 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:47:07,226 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:47:07,227 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:07,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:47:07,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:47:07,240 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store
2024-11-20T06:47:07,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:47:07,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:47:07,253 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:07,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:47:07,254 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:07,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:07,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:47:07,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:07,254 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:07,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085227254Disabling compacts and flushes for region at 1732085227254Disabling writes for close at 1732085227254Writing region close event to WAL at 1732085227254Closed at 1732085227254
2024-11-20T06:47:07,255 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/.initializing
2024-11-20T06:47:07,255 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,258 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C38687%2C1732085226879, suffix=, logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879, archiveDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/oldWALs, maxLogs=10
2024-11-20T06:47:07,259 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C38687%2C1732085226879.1732085227258
2024-11-20T06:47:07,267 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258
2024-11-20T06:47:07,275 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36513:36513),(127.0.0.1/127.0.0.1:40221:40221)]
2024-11-20T06:47:07,278 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:47:07,279 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:07,279 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,279 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,281 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:07,282 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:47:07,282 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,283 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:07,283 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:07,285 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:47:07,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,286 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:07,286 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:07,287 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:47:07,287 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,288 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:07,288 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:47:07,290 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:47:07,290 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,291 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:07,291 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,292 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,293 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,294 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,294 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,295 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:47:07,296 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:47:07,303 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:47:07,304 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=714889, jitterRate=-0.09097190201282501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:47:07,305 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085227279Initializing all the Stores at 1732085227280 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085227280Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085227280Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085227280Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085227281 (+1 ms)Cleaning up temporary data from old regions at 1732085227294 (+13 ms)Region opened successfully at 1732085227305 (+11 ms)
2024-11-20T06:47:07,306 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:47:07,311 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@116ffc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:47:07,312 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:47:07,312 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:47:07,312 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:47:07,312 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:47:07,313 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:47:07,313 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:47:07,313 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:47:07,321 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:47:07,322 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:47:07,352 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:47:07,352 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:47:07,353 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:47:07,363 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:47:07,363 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:47:07,365 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:47:07,371 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:47:07,372 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:47:07,379 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:47:07,382 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:47:07,388 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:47:07,396 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:07,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:07,396 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,397 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,38687,1732085226879, sessionid=0x1015848b02d0000, setting cluster-up flag (Was=false)
2024-11-20T06:47:07,413 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,438 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:47:07,439 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,455 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,480 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:47:07,481 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:07,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:47:07,488 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:07,489 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:47:07,489 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:47:07,489 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,38687,1732085226879
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:47:07,490 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:47:07,491 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,496 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:07,497 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:47:07,498 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,499 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:47:07,502 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085257502
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:47:07,503 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:47:07,507 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,507 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:47:07,508 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:47:07,508 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:47:07,508 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:47:07,508 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:47:07,508 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085227508,5,FailOnTimeoutGroup]
2024-11-20T06:47:07,509 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085227508,5,FailOnTimeoutGroup]
2024-11-20T06:47:07,509 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,509 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:47:07,509 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,509 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:47:07,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:47:07,513 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:47:07,514 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a
2024-11-20T06:47:07,517 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(746): ClusterId : 272846e8-f98b-42e6-8b48-4a58ad6b25d4
2024-11-20T06:47:07,517 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:47:07,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:47:07,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:47:07,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:07,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:47:07,527 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:47:07,527 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:47:07,527 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:47:07,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:07,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:47:07,530 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:47:07,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:07,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:47:07,533 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:47:07,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,534 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:07,534 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:47:07,536 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:47:07,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:07,536 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:07,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:47:07,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740
2024-11-20T06:47:07,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740
2024-11-20T06:47:07,539 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:47:07,539 DEBUG [RS:0;61c2066a8a0f:41993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d3ec26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:47:07,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:47:07,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:47:07,543 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:47:07,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:47:07,547 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:47:07,547 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=761559, jitterRate=-0.031627655029296875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:47:07,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085227524Initializing all the Stores at 1732085227525 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085227525Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085227526 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085227526Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085227526Cleaning up temporary data from old regions at 1732085227539 (+13 ms)Region opened successfully at 1732085227548 (+9 ms)
2024-11-20T06:47:07,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:47:07,548 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:47:07,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:47:07,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:47:07,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:47:07,549 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:07,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085227548Disabling compacts and flushes for region at 1732085227548Disabling writes for close at 1732085227548Writing region close event to WAL at 1732085227549 (+1 ms)Closed at 1732085227549
2024-11-20T06:47:07,550 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:07,551 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:47:07,551 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:47:07,552 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:47:07,554 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:41993
2024-11-20T06:47:07,554 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:47:07,554 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:47:07,554 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:47:07,554 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:47:07,555 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,38687,1732085226879 with port=41993, startcode=1732085227061
2024-11-20T06:47:07,555 DEBUG [RS:0;61c2066a8a0f:41993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:47:07,557 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:47:07,557 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38687 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,558 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38687 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,559 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a
2024-11-20T06:47:07,559 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45393
2024-11-20T06:47:07,559 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:47:07,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:07,572 DEBUG [RS:0;61c2066a8a0f:41993 {}] zookeeper.ZKUtil(111): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,572 WARN  [RS:0;61c2066a8a0f:41993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:47:07,572 INFO  [RS:0;61c2066a8a0f:41993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:07,572 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,572 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,41993,1732085227061]
2024-11-20T06:47:07,576 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:47:07,579 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:47:07,580 INFO  [RS:0;61c2066a8a0f:41993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:47:07,580 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,580 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:47:07,581 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:47:07,581 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,581 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,581 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:07,582 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:07,583 DEBUG [RS:0;61c2066a8a0f:41993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,587 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41993,1732085227061-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:47:07,603 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:47:07,603 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41993,1732085227061-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,603 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,603 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.Replication(171): 61c2066a8a0f,41993,1732085227061 started
2024-11-20T06:47:07,617 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:07,617 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,41993,1732085227061, RpcServer on 61c2066a8a0f/172.17.0.2:41993, sessionid=0x1015848b02d0001
2024-11-20T06:47:07,617 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:47:07,618 DEBUG [RS:0;61c2066a8a0f:41993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,618 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41993,1732085227061'
2024-11-20T06:47:07,618 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:47:07,618 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41993,1732085227061'
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:47:07,619 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:47:07,620 DEBUG [RS:0;61c2066a8a0f:41993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:47:07,620 INFO  [RS:0;61c2066a8a0f:41993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:47:07,620 INFO  [RS:0;61c2066a8a0f:41993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:47:07,704 WARN  [61c2066a8a0f:38687 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:47:07,722 INFO  [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41993%2C1732085227061, suffix=, logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061, archiveDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs, maxLogs=32
2024-11-20T06:47:07,723 INFO  [RS:0;61c2066a8a0f:41993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41993%2C1732085227061.1732085227723
2024-11-20T06:47:07,754 INFO  [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
2024-11-20T06:47:07,758 DEBUG [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36513:36513),(127.0.0.1/127.0.0.1:40221:40221)]
2024-11-20T06:47:07,954 DEBUG [61c2066a8a0f:38687 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:47:07,955 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:07,957 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,41993,1732085227061, state=OPENING
2024-11-20T06:47:07,971 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:47:07,979 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:07,980 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:47:07,980 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41993,1732085227061}]
2024-11-20T06:47:07,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:07,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:08,134 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:47:08,136 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40291, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:47:08,139 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:47:08,140 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:08,142 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41993%2C1732085227061.meta, suffix=.meta, logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061, archiveDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs, maxLogs=32
2024-11-20T06:47:08,142 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
2024-11-20T06:47:08,148 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
2024-11-20T06:47:08,151 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36513:36513),(127.0.0.1/127.0.0.1:40221:40221)]
2024-11-20T06:47:08,154 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:47:08,155 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:47:08,155 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:47:08,158 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:47:08,159 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:47:08,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:08,159 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:08,159 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:47:08,160 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:47:08,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:08,161 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:08,161 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:47:08,161 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:47:08,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:08,162 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:08,162 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:47:08,163 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:47:08,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:08,163 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:47:08,163 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:47:08,164 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740
2024-11-20T06:47:08,165 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740
2024-11-20T06:47:08,166 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:47:08,166 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:47:08,167 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:47:08,168 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:47:08,169 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811323, jitterRate=0.03165125846862793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:47:08,169 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:47:08,170 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085228155Writing region info on filesystem at 1732085228155Initializing all the Stores at 1732085228156 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085228156Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085228158 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085228158Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085228158Cleaning up temporary data from old regions at 1732085228166 (+8 ms)Running coprocessor post-open hooks at 1732085228169 (+3 ms)Region opened successfully at 1732085228170 (+1 ms)
2024-11-20T06:47:08,171 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085228133
2024-11-20T06:47:08,174 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:47:08,174 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:47:08,175 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:08,176 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,41993,1732085227061, state=OPEN
2024-11-20T06:47:08,249 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:47:08,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:47:08,249 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:08,249 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:08,249 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:47:08,253 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:47:08,253 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41993,1732085227061 in 269 msec
2024-11-20T06:47:08,257 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:47:08,257 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 702 msec
2024-11-20T06:47:08,258 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:47:08,258 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:47:08,259 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:47:08,259 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,41993,1732085227061, seqNum=-1]
2024-11-20T06:47:08,260 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:47:08,261 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:47:08,268 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 779 msec
2024-11-20T06:47:08,269 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085228268, completionTime=-1
2024-11-20T06:47:08,269 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:47:08,269 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:47:08,271 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:47:08,271 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085288271
2024-11-20T06:47:08,271 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085348271
2024-11-20T06:47:08,271 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:38687, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,272 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,274 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.112sec
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:47:08,277 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:47:08,280 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:47:08,280 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:47:08,280 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38687,1732085226879-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb75251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:08,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,38687,-1 for getting cluster id
2024-11-20T06:47:08,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:47:08,320 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '272846e8-f98b-42e6-8b48-4a58ad6b25d4'
2024-11-20T06:47:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:47:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "272846e8-f98b-42e6-8b48-4a58ad6b25d4"

2024-11-20T06:47:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11243140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,38687,-1]
2024-11-20T06:47:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:47:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:08,323 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:47:08,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451ba750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:47:08,324 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:47:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,41993,1732085227061, seqNum=-1]
2024-11-20T06:47:08,325 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:47:08,327 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:47:08,329 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:08,329 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:08,332 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:08,346 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:08,347 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:47:08,347 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:08,347 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41921
2024-11-20T06:47:08,349 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41921 connecting to ZooKeeper ensemble=127.0.0.1:62382
2024-11-20T06:47:08,349 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:08,351 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:08,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419210x0, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:08,372 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-20T06:47:08,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:419210x0, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-20T06:47:08,372 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41921-0x1015848b02d0002 connected
2024-11-20T06:47:08,374 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:47:08,375 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:47:08,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:47:08,378 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:08,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41921
2024-11-20T06:47:08,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41921
2024-11-20T06:47:08,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41921
2024-11-20T06:47:08,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41921
2024-11-20T06:47:08,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41921
2024-11-20T06:47:08,382 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(746): ClusterId : 272846e8-f98b-42e6-8b48-4a58ad6b25d4
2024-11-20T06:47:08,382 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:47:08,394 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:47:08,394 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:47:08,405 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:47:08,406 DEBUG [RS:1;61c2066a8a0f:41921 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e9e7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:47:08,418 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;61c2066a8a0f:41921
2024-11-20T06:47:08,418 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:47:08,418 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:47:08,418 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:47:08,419 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,38687,1732085226879 with port=41921, startcode=1732085228346
2024-11-20T06:47:08,419 DEBUG [RS:1;61c2066a8a0f:41921 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:47:08,420 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:47:08,421 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38687 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,421 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38687 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,422 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a
2024-11-20T06:47:08,422 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45393
2024-11-20T06:47:08,423 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:47:08,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:08,430 DEBUG [RS:1;61c2066a8a0f:41921 {}] zookeeper.ZKUtil(111): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,430 WARN  [RS:1;61c2066a8a0f:41921 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:47:08,430 INFO  [RS:1;61c2066a8a0f:41921 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:47:08,430 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,41921,1732085228346]
2024-11-20T06:47:08,430 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,434 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:47:08,436 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:47:08,437 INFO  [RS:1;61c2066a8a0f:41921 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:47:08,437 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,437 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:47:08,438 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:47:08,438 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,439 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,440 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,440 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:47:08,440 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:08,440 DEBUG [RS:1;61c2066a8a0f:41921 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:47:08,440 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,441 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,441 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,441 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,441 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,441 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41921,1732085228346-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:47:08,454 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:47:08,455 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41921,1732085228346-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,455 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,455 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.Replication(171): 61c2066a8a0f,41921,1732085228346 started
2024-11-20T06:47:08,467 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:47:08,467 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,41921,1732085228346, RpcServer on 61c2066a8a0f/172.17.0.2:41921, sessionid=0x1015848b02d0002
2024-11-20T06:47:08,468 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:47:08,468 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;61c2066a8a0f:41921,5,FailOnTimeoutGroup]
2024-11-20T06:47:08,468 DEBUG [RS:1;61c2066a8a0f:41921 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,468 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41921,1732085228346'
2024-11-20T06:47:08,468 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:47:08,468 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-20T06:47:08,468 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:47:08,468 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41921,1732085228346'
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:47:08,469 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:47:08,470 DEBUG [RS:1;61c2066a8a0f:41921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:47:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:08,470 INFO  [RS:1;61c2066a8a0f:41921 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:47:08,470 INFO  [RS:1;61c2066a8a0f:41921 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:47:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bc8811f
2024-11-20T06:47:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T06:47:08,471 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T06:47:08,472 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T06:47:08,472 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T06:47:08,473 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:47:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T06:47:08,476 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T06:47:08,476 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:08,476 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-20T06:47:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:47:08,477 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T06:47:08,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741835_1011 (size=393)
2024-11-20T06:47:08,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741835_1011 (size=393)
2024-11-20T06:47:08,486 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f30a91b3b12ffdaf1928b202e5ace79, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a
2024-11-20T06:47:08,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37307 is added to blk_1073741836_1012 (size=76)
2024-11-20T06:47:08,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39949 is added to blk_1073741836_1012 (size=76)
2024-11-20T06:47:08,573 INFO  [RS:1;61c2066a8a0f:41921 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41921%2C1732085228346, suffix=, logDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346, archiveDir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs, maxLogs=32
2024-11-20T06:47:08,575 INFO  [RS:1;61c2066a8a0f:41921 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085228575
2024-11-20T06:47:08,584 INFO  [RS:1;61c2066a8a0f:41921 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575
2024-11-20T06:47:08,590 DEBUG [RS:1;61c2066a8a0f:41921 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36513:36513),(127.0.0.1/127.0.0.1:40221:40221)]
2024-11-20T06:47:08,894 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:08,894 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 7f30a91b3b12ffdaf1928b202e5ace79, disabling compactions & flushes
2024-11-20T06:47:08,894 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:08,895 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:08,895 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79. after waiting 0 ms
2024-11-20T06:47:08,895 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:08,895 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:08,895 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f30a91b3b12ffdaf1928b202e5ace79:
Waiting for close lock at 1732085228894Disabling compacts and flushes for region at 1732085228894Disabling writes for close at 1732085228895 (+1 ms)Writing region close event to WAL at 1732085228895Closed at 1732085228895
2024-11-20T06:47:08,897 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T06:47:08,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732085228897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085228897"}]},"ts":"1732085228897"}
2024-11-20T06:47:08,900 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T06:47:08,901 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T06:47:08,901 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085228901"}]},"ts":"1732085228901"}
2024-11-20T06:47:08,904 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-20T06:47:08,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0}
2024-11-20T06:47:08,910 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-20T06:47:08,910 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-20T06:47:08,910 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-20T06:47:08,910 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-20T06:47:08,910 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-20T06:47:08,910 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-20T06:47:08,910 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-20T06:47:08,911 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7f30a91b3b12ffdaf1928b202e5ace79, ASSIGN}]
2024-11-20T06:47:08,912 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7f30a91b3b12ffdaf1928b202e5ace79, ASSIGN
2024-11-20T06:47:08,913 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7f30a91b3b12ffdaf1928b202e5ace79, ASSIGN; state=OFFLINE, location=61c2066a8a0f,41921,1732085228346; forceNewPlan=false, retain=false
2024-11-20T06:47:09,064 INFO  [61c2066a8a0f:38687 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-20T06:47:09,065 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7f30a91b3b12ffdaf1928b202e5ace79, regionState=OPENING, regionLocation=61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:09,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7f30a91b3b12ffdaf1928b202e5ace79, ASSIGN because future has completed
2024-11-20T06:47:09,068 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f30a91b3b12ffdaf1928b202e5ace79, server=61c2066a8a0f,41921,1732085228346}]
2024-11-20T06:47:09,222 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:47:09,226 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:47:09,231 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:09,231 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f30a91b3b12ffdaf1928b202e5ace79, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:47:09,232 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,232 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:47:09,232 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,232 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,234 INFO  [StoreOpener-7f30a91b3b12ffdaf1928b202e5ace79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7f30a91b3b12ffdaf1928b202e5ace79 
2024-11-20T06:47:09,236 INFO  [StoreOpener-7f30a91b3b12ffdaf1928b202e5ace79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f30a91b3b12ffdaf1928b202e5ace79 columnFamilyName info
2024-11-20T06:47:09,236 DEBUG [StoreOpener-7f30a91b3b12ffdaf1928b202e5ace79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:47:09,236 INFO  [StoreOpener-7f30a91b3b12ffdaf1928b202e5ace79-1 {}] regionserver.HStore(327): Store=7f30a91b3b12ffdaf1928b202e5ace79/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:47:09,237 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,238 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,238 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,238 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,238 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,240 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,242 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:47:09,243 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7f30a91b3b12ffdaf1928b202e5ace79; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=853291, jitterRate=0.08501660823822021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:47:09,243 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:09,244 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7f30a91b3b12ffdaf1928b202e5ace79:
Running coprocessor pre-open hook at 1732085229232Writing region info on filesystem at 1732085229232Initializing all the Stores at 1732085229233 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085229233Cleaning up temporary data from old regions at 1732085229239 (+6 ms)Running coprocessor post-open hooks at 1732085229243 (+4 ms)Region opened successfully at 1732085229244 (+1 ms)
2024-11-20T06:47:09,245 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79., pid=6, masterSystemTime=1732085229222
2024-11-20T06:47:09,248 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:09,248 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:09,249 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7f30a91b3b12ffdaf1928b202e5ace79, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:09,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f30a91b3b12ffdaf1928b202e5ace79, server=61c2066a8a0f,41921,1732085228346 because future has completed
2024-11-20T06:47:09,257 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T06:47:09,257 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f30a91b3b12ffdaf1928b202e5ace79, server=61c2066a8a0f,41921,1732085228346 in 186 msec
2024-11-20T06:47:09,261 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T06:47:09,261 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=7f30a91b3b12ffdaf1928b202e5ace79, ASSIGN in 346 msec
2024-11-20T06:47:09,262 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T06:47:09,263 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085229263"}]},"ts":"1732085229263"}
2024-11-20T06:47:09,266 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-20T06:47:09,269 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T06:47:09,272 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 796 msec
2024-11-20T06:47:13,657 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:47:13,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:13,679 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:13,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:13,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:13,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:47:13,689 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T06:47:13,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T06:47:13,689 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-20T06:47:13,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:47:13,690 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T06:47:14,434 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-20T06:47:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:47:18,552 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-20T06:47:18,552 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-20T06:47:18,558 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T06:47:18,558 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:18,573 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:18,576 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:18,576 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:18,576 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:18,576 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:47:18,577 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cac6b83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:18,577 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc4ef73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:18,672 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65429201{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-45837-hadoop-hdfs-3_4_1-tests_jar-_-any-15936217771459712941/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:18,672 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4cbb1003{HTTP/1.1, (http/1.1)}{localhost:45837}
2024-11-20T06:47:18,672 INFO  [Time-limited test {}] server.Server(415): Started @122014ms
2024-11-20T06:47:18,673 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:18,701 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:18,705 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:18,705 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:18,705 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:18,706 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:18,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3eb2cdd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:18,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bea65f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:18,801 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70a0068f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-37325-hadoop-hdfs-3_4_1-tests_jar-_-any-5773008514147475625/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:18,802 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39615ad8{HTTP/1.1, (http/1.1)}{localhost:37325}
2024-11-20T06:47:18,802 INFO  [Time-limited test {}] server.Server(415): Started @122144ms
2024-11-20T06:47:18,803 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:18,832 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:18,835 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:18,835 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:18,835 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:18,835 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:18,836 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264a677b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:18,836 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@158a9d8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:18,931 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ec5aa68{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-41463-hadoop-hdfs-3_4_1-tests_jar-_-any-497396308173536068/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:18,931 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@393eb51f{HTTP/1.1, (http/1.1)}{localhost:41463}
2024-11-20T06:47:18,931 INFO  [Time-limited test {}] server.Server(415): Started @122273ms
2024-11-20T06:47:18,932 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:19,718 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data5/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,719 WARN  [Thread-870 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data6/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,744 WARN  [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:19,746 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e212c8bc52750e9 with lease ID 0xd87bea97461ac234: Processing first storage report for DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94 from datanode DatanodeRegistration(127.0.0.1:33321, datanodeUuid=76903bef-0dcb-4365-9e3d-2b4cd818521c, infoPort=46105, infoSecurePort=0, ipcPort=45445, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,746 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e212c8bc52750e9 with lease ID 0xd87bea97461ac234: from storage DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94 node DatanodeRegistration(127.0.0.1:33321, datanodeUuid=76903bef-0dcb-4365-9e3d-2b4cd818521c, infoPort=46105, infoSecurePort=0, ipcPort=45445, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,746 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e212c8bc52750e9 with lease ID 0xd87bea97461ac234: Processing first storage report for DS-254fc2b3-9a2f-4ab5-b0a9-f3e2c3aba225 from datanode DatanodeRegistration(127.0.0.1:33321, datanodeUuid=76903bef-0dcb-4365-9e3d-2b4cd818521c, infoPort=46105, infoSecurePort=0, ipcPort=45445, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,746 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e212c8bc52750e9 with lease ID 0xd87bea97461ac234: from storage DS-254fc2b3-9a2f-4ab5-b0a9-f3e2c3aba225 node DatanodeRegistration(127.0.0.1:33321, datanodeUuid=76903bef-0dcb-4365-9e3d-2b4cd818521c, infoPort=46105, infoSecurePort=0, ipcPort=45445, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,827 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,827 WARN  [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,848 WARN  [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:19,851 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20e87f6310c2ee7 with lease ID 0xd87bea97461ac235: Processing first storage report for DS-69db36d5-f018-443a-8c9c-853b6bf56aec from datanode DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,851 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20e87f6310c2ee7 with lease ID 0xd87bea97461ac235: from storage DS-69db36d5-f018-443a-8c9c-853b6bf56aec node DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,851 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20e87f6310c2ee7 with lease ID 0xd87bea97461ac235: Processing first storage report for DS-745ee08e-5eee-4d2a-aa16-3a0a7af741ad from datanode DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,851 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20e87f6310c2ee7 with lease ID 0xd87bea97461ac235: from storage DS-745ee08e-5eee-4d2a-aa16-3a0a7af741ad node DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,868 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data9/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,868 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data10/current/BP-127923120-172.17.0.2-1732085225158/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:19,888 WARN  [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:19,890 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x583ad0d9c4408332 with lease ID 0xd87bea97461ac236: Processing first storage report for DS-b7eed287-7d6a-44f6-aad2-cb955120661c from datanode DatanodeRegistration(127.0.0.1:42755, datanodeUuid=f9bb743a-2131-41ec-9cad-4451de61eb9a, infoPort=42523, infoSecurePort=0, ipcPort=37105, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,891 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x583ad0d9c4408332 with lease ID 0xd87bea97461ac236: from storage DS-b7eed287-7d6a-44f6-aad2-cb955120661c node DatanodeRegistration(127.0.0.1:42755, datanodeUuid=f9bb743a-2131-41ec-9cad-4451de61eb9a, infoPort=42523, infoSecurePort=0, ipcPort=37105, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,891 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x583ad0d9c4408332 with lease ID 0xd87bea97461ac236: Processing first storage report for DS-b356e9b1-77f1-4c78-b1cb-816d848a4fa7 from datanode DatanodeRegistration(127.0.0.1:42755, datanodeUuid=f9bb743a-2131-41ec-9cad-4451de61eb9a, infoPort=42523, infoSecurePort=0, ipcPort=37105, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158)
2024-11-20T06:47:19,891 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x583ad0d9c4408332 with lease ID 0xd87bea97461ac236: from storage DS-b356e9b1-77f1-4c78-b1cb-816d848a4fa7 node DatanodeRegistration(127.0.0.1:42755, datanodeUuid=f9bb743a-2131-41ec-9cad-4451de61eb9a, infoPort=42523, infoSecurePort=0, ipcPort=37105, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:19,958 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,958 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,958 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,959 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,959 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 block BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:19,959 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 block BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:19,959 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta block BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:19,959 WARN  [PacketResponder: BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39949] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,959 WARN  [PacketResponder: BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39949] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,959 WARN  [PacketResponder: BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39949] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,959 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 block BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:19,960 WARN  [PacketResponder: BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39949] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:57016 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:37307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57016 dst: /127.0.0.1:37307
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329950404_22 at /127.0.0.1:57008 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57008 dst: /127.0.0.1:37307
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1817268510_22 at /127.0.0.1:56972 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56972 dst: /127.0.0.1:37307
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329950404_22 at /127.0.0.1:47596 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39949:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47596 dst: /127.0.0.1:39949
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1817268510_22 at /127.0.0.1:47558 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39949:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47558 dst: /127.0.0.1:39949
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329950404_22 at /127.0.0.1:56996 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37307:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56996 dst: /127.0.0.1:37307
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,963 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:47618 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:39949:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47618 dst: /127.0.0.1:39949
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,963 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@272348fe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:19,963 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329950404_22 at /127.0.0.1:47584 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39949:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47584 dst: /127.0.0.1:39949
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,964 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10b53169{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:19,965 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:19,965 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bb19ef9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:19,965 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32403ac6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:19,966 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:19,966 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:19,966 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid d67726cb-0b7c-4be9-93ad-8ddfa3123cc6) service to localhost/127.0.0.1:45393
2024-11-20T06:47:19,966 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:19,967 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data3/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:19,967 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data4/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:19,967 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:19,970 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@f1c722b {}] datanode.DataXceiver(331): 127.0.0.1:37307:DataXceiver error processing unknown operation  src: /127.0.0.1:42270 dst: /127.0.0.1:37307
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:19,970 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 block BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,970 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta block BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,971 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 block BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,975 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 block BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5538b075{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:19,980 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f87a993{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:19,980 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:19,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bb5d847{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:19,980 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf32f74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:19,981 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:19,981 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:19,982 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid a306182b-ad6d-4234-b341-832e7ffd241e) service to localhost/127.0.0.1:45393
2024-11-20T06:47:19,982 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:19,982 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data1/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:19,982 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data2/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:19,982 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79., hostname=61c2066a8a0f,41921,1732085228346, seqNum=2]
2024-11-20T06:47:19,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:47:19,988 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:47:19,989 ERROR [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a-prefix:61c2066a8a0f,41921,1732085228346 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,989 WARN  [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a-prefix:61c2066a8a0f,41921,1732085228346 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,989 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C41921%2C1732085228346:(num 1732085228575) roll requested
2024-11-20T06:47:19,989 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085239989
2024-11-20T06:47:19,991 WARN  [Thread-903 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:19,992 WARN  [Thread-903 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:19,992 WARN  [Thread-903 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741838_1018
2024-11-20T06:47:19,994 WARN  [Thread-903 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:19,999 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:20,000 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:20,000 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:20,000 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:20,000 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:20,000 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989
2024-11-20T06:47:20,000 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:20,001 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:20,001 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42523:42523),(127.0.0.1/127.0.0.1:46105:46105)]
2024-11-20T06:47:20,001 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:20,002 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-20T06:47:20,002 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-20T06:47:20,002 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575
2024-11-20T06:47:20,005 WARN  [IPC Server handler 4 on default port 45393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1013
2024-11-20T06:47:20,010 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 after 5ms
2024-11-20T06:47:21,228 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:21,584 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:21,584 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:22,003 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989
2024-11-20T06:47:22,004 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:22,004 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989 block BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:22,005 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:46922 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:42755:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46922 dst: /127.0.0.1:42755
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:22,005 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:44382 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44382 dst: /127.0.0.1:33321
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:22,046 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ec5aa68{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:22,047 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@393eb51f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:22,047 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:22,047 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@158a9d8c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:22,047 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264a677b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:22,048 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:22,048 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:22,048 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid f9bb743a-2131-41ec-9cad-4451de61eb9a) service to localhost/127.0.0.1:45393
2024-11-20T06:47:22,048 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:22,048 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data9/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:22,049 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data10/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:22,049 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:23,228 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:23,585 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:23,585 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:24,002 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]
2024-11-20T06:47:24,002 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C41921%2C1732085228346:(num 1732085239989) roll requested
2024-11-20T06:47:24,003 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085244003
2024-11-20T06:47:24,011 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:24,011 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:24,011 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741840_1022
2024-11-20T06:47:24,011 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 after 4009ms
2024-11-20T06:47:24,012 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:24,013 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:24,013 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:24,013 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741841_1023
2024-11-20T06:47:24,013 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:24,015 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:24,015 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:24,015 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741842_1024
2024-11-20T06:47:24,015 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:24,020 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:24,020 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:24,020 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:24,020 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:24,020 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:24,020 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085244003
2024-11-20T06:47:24,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741839_1021 (size=2431)
2024-11-20T06:47:24,027 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46105:46105),(127.0.0.1/127.0.0.1:46139:46139)]
2024-11-20T06:47:24,027 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:24,027 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989 is not closed yet, will try archiving it next time
2024-11-20T06:47:24,054 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-20T06:47:24,424 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:25,229 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:25,585 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:25,586 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,058 WARN  [ResponseProcessor for block BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,059 WARN  [DataStreamer for file /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085244003 block BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:26,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:35562 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35562 dst: /127.0.0.1:33321
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:26,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42150 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42150 dst: /127.0.0.1:45705
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:26,107 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65429201{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:26,107 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4cbb1003{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:26,107 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:26,108 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc4ef73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:26,108 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cac6b83{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:26,110 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:26,110 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:26,110 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:26,110 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid 76903bef-0dcb-4365-9e3d-2b4cd818521c) service to localhost/127.0.0.1:45393
2024-11-20T06:47:26,111 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data5/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:26,111 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data6/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:26,111 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:26,120 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:47:26,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/c3b0002228e0486aac32db3e32251ab2 is 1080, key is row0002/info:/1732085242050/Put/seqid=0
2024-11-20T06:47:26,141 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,141 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:26,141 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741844_1027
2024-11-20T06:47:26,142 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:26,143 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,143 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:26,143 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741845_1028
2024-11-20T06:47:26,143 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:26,145 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,145 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:26,145 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741846_1029
2024-11-20T06:47:26,145 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:26,148 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,148 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42172 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:26,148 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:26,148 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030
2024-11-20T06:47:26,148 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42172 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:26,148 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42172 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42172 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:26,148 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:26,149 WARN  [IPC Server handler 4 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:26,149 WARN  [IPC Server handler 4 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:26,150 WARN  [IPC Server handler 4 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:26,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741848_1031 (size=10347)
2024-11-20T06:47:26,554 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/c3b0002228e0486aac32db3e32251ab2
2024-11-20T06:47:26,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/c3b0002228e0486aac32db3e32251ab2 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2
2024-11-20T06:47:26,573 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2, entries=5, sequenceid=11, filesize=10.1 K
2024-11-20T06:47:26,574 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 7f30a91b3b12ffdaf1928b202e5ace79 in 454ms, sequenceid=11, compaction requested=false
2024-11-20T06:47:26,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:26,745 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-20T06:47:26,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/b9051486efe0471d8caddd6a1fb82e11 is 1080, key is row0007/info:/1732085246121/Put/seqid=0
2024-11-20T06:47:26,752 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,752 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:26,752 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741849_1032
2024-11-20T06:47:26,753 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:26,754 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,754 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:26,754 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741850_1033
2024-11-20T06:47:26,755 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:26,756 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,757 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:26,757 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741851_1034
2024-11-20T06:47:26,758 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:26,759 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:26,759 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:26,759 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741852_1035
2024-11-20T06:47:26,761 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:26,761 WARN  [IPC Server handler 4 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:26,762 WARN  [IPC Server handler 4 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:26,762 WARN  [IPC Server handler 4 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:26,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741853_1036 (size=12506)
2024-11-20T06:47:26,860 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d44a2d7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741848_1031 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:27,169 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/b9051486efe0471d8caddd6a1fb82e11
2024-11-20T06:47:27,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/b9051486efe0471d8caddd6a1fb82e11 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11
2024-11-20T06:47:27,190 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11, entries=7, sequenceid=24, filesize=12.2 K
2024-11-20T06:47:27,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 7f30a91b3b12ffdaf1928b202e5ace79 in 447ms, sequenceid=24, compaction requested=false
2024-11-20T06:47:27,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:27,192 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-20T06:47:27,192 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:27,192 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11 because midkey is the same as first or last row
2024-11-20T06:47:27,229 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:27,586 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:27,586 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,027 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]
2024-11-20T06:47:28,027 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C41921%2C1732085228346:(num 1732085244003) roll requested
2024-11-20T06:47:28,028 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085248027
2024-11-20T06:47:28,031 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,031 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:28,031 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741854_1037
2024-11-20T06:47:28,032 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:28,033 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,033 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:28,033 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741855_1038
2024-11-20T06:47:28,034 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:28,036 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42755
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,036 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42210 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039 to mirror 127.0.0.1:42755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,037 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:28,037 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039
2024-11-20T06:47:28,037 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42210 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T06:47:28,037 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42210 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42210 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,037 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:28,040 WARN  [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39949
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42226 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040 to mirror 127.0.0.1:39949
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,040 WARN  [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:28,040 WARN  [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040
2024-11-20T06:47:28,040 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42226 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T06:47:28,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42226 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42226 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,041 WARN  [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:28,041 WARN  [IPC Server handler 3 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:28,041 WARN  [IPC Server handler 3 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:28,042 WARN  [IPC Server handler 3 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:28,045 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:28,045 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:28,045 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:28,045 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:28,045 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:28,045 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085244003 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085248027
2024-11-20T06:47:28,046 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46139:46139)]
2024-11-20T06:47:28,046 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:28,046 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085244003 is not closed yet, will try archiving it next time
2024-11-20T06:47:28,046 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085239989 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs/61c2066a8a0f%2C41921%2C1732085228346.1732085239989
2024-11-20T06:47:28,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741843_1026 (size=25992)
2024-11-20T06:47:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:28,181 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T06:47:28,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/463e5f1646f644fc8c99480777208f95 is 1079, key is tmprow/info:/1732085248179/Put/seqid=0
2024-11-20T06:47:28,192 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,192 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:28,192 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741859_1042
2024-11-20T06:47:28,193 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:28,194 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,194 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:28,194 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741860_1043
2024-11-20T06:47:28,195 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:28,196 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,196 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:28,196 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741861_1044
2024-11-20T06:47:28,197 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:28,199 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,199 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42236 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,199 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:28,200 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045
2024-11-20T06:47:28,200 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42236 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:28,200 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42236 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42236 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,200 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:28,201 WARN  [IPC Server handler 2 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:28,201 WARN  [IPC Server handler 2 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:28,201 WARN  [IPC Server handler 2 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:28,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741863_1046 (size=6027)
2024-11-20T06:47:28,448 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:28,606 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/463e5f1646f644fc8c99480777208f95
2024-11-20T06:47:28,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/463e5f1646f644fc8c99480777208f95 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95
2024-11-20T06:47:28,626 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95, entries=1, sequenceid=34, filesize=5.9 K
2024-11-20T06:47:28,628 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7f30a91b3b12ffdaf1928b202e5ace79 in 446ms, sequenceid=34, compaction requested=true
2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11 because midkey is the same as first or last row
2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f30a91b3b12ffdaf1928b202e5ace79:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:47:28,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:28,628 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:47:28,630 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:47:28,630 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1541): 7f30a91b3b12ffdaf1928b202e5ace79/info is initiating minor compaction (all files)
2024-11-20T06:47:28,630 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f30a91b3b12ffdaf1928b202e5ace79/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:28,630 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95] into tmpdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp, totalSize=28.2 K
2024-11-20T06:47:28,630 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b0002228e0486aac32db3e32251ab2, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732085242050
2024-11-20T06:47:28,631 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9051486efe0471d8caddd6a1fb82e11, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732085246121
2024-11-20T06:47:28,631 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting 463e5f1646f644fc8c99480777208f95, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732085248179
2024-11-20T06:47:28,644 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f30a91b3b12ffdaf1928b202e5ace79#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:47:28,645 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/84756580f4494a84b2f807cb98112c8b is 1080, key is row0002/info:/1732085242050/Put/seqid=0
2024-11-20T06:47:28,646 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,647 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:28,647 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741864_1047
2024-11-20T06:47:28,647 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:28,648 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,649 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:28,649 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741865_1048
2024-11-20T06:47:28,649 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:28,651 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37307
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,651 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42280 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049 to mirror 127.0.0.1:37307
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,651 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:28,651 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049
2024-11-20T06:47:28,651 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42280 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:28,651 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42280 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42280 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:28,652 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:28,653 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:28,653 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:28,653 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741867_1050
2024-11-20T06:47:28,654 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:28,654 WARN  [IPC Server handler 1 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:28,654 WARN  [IPC Server handler 1 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:28,655 WARN  [IPC Server handler 1 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:28,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741868_1051 (size=17994)
2024-11-20T06:47:29,065 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/84756580f4494a84b2f807cb98112c8b as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b
2024-11-20T06:47:29,073 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f30a91b3b12ffdaf1928b202e5ace79/info of 7f30a91b3b12ffdaf1928b202e5ace79 into 84756580f4494a84b2f807cb98112c8b(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:29,073 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79., storeName=7f30a91b3b12ffdaf1928b202e5ace79/info, priority=13, startTime=1732085248628; duration=0sec
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b because midkey is the same as first or last row
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b because midkey is the same as first or last row
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:29,073 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b because midkey is the same as first or last row
2024-11-20T06:47:29,074 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:29,074 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f30a91b3b12ffdaf1928b202e5ace79:info
2024-11-20T06:47:29,230 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,587 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,587 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:29,612 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T06:47:29,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/8cd9e014241444abb1ccb0842577cedb is 1079, key is tmprow/info:/1732085249610/Put/seqid=0
2024-11-20T06:47:29,621 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,622 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:29,622 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741869_1052
2024-11-20T06:47:29,622 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:29,624 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,624 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:29,624 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741870_1053
2024-11-20T06:47:29,625 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:29,626 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,626 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:29,626 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741871_1054
2024-11-20T06:47:29,627 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:29,629 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:29,629 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42292 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:29,629 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:29,630 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055
2024-11-20T06:47:29,630 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42292 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:29,630 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42292 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741872_1055] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42292 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:29,630 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:29,631 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:29,631 WARN  [IPC Server handler 0 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:29,631 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:29,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741873_1056 (size=6027)
2024-11-20T06:47:29,852 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d44a2d7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741853_1036 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:29,852 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@265fafc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741843_1026 to 127.0.0.1:33321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,035 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/8cd9e014241444abb1ccb0842577cedb
2024-11-20T06:47:30,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/8cd9e014241444abb1ccb0842577cedb as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb
2024-11-20T06:47:30,046 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]
2024-11-20T06:47:30,046 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C41921%2C1732085228346:(num 1732085248027) roll requested
2024-11-20T06:47:30,047 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085250047
2024-11-20T06:47:30,052 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:30,052 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42304 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,052 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:30,052 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057
2024-11-20T06:47:30,052 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42304 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T06:47:30,052 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb, entries=1, sequenceid=45, filesize=5.9 K
2024-11-20T06:47:30,052 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42304 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42304 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,053 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:30,054 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7f30a91b3b12ffdaf1928b202e5ace79 in 441ms, sequenceid=45, compaction requested=false
2024-11-20T06:47:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-20T06:47:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b because midkey is the same as first or last row
2024-11-20T06:47:30,054 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:30,054 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:30,055 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741875_1058
2024-11-20T06:47:30,055 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:30,057 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37307
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:30,057 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42306 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059 to mirror 127.0.0.1:37307
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,058 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:30,058 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059
2024-11-20T06:47:30,058 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42306 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-20T06:47:30,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42306 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42306 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,058 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:30,060 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:30,060 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:30,060 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741877_1060
2024-11-20T06:47:30,060 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:30,061 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:30,061 WARN  [IPC Server handler 0 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:30,061 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:30,063 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:30,063 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:30,063 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:30,064 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:30,064 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:30,064 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085248027 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085250047
2024-11-20T06:47:30,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741858_1041 (size=13591)
2024-11-20T06:47:30,066 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46139:46139)]
2024-11-20T06:47:30,066 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:30,066 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085248027 is not closed yet, will try archiving it next time
2024-11-20T06:47:30,066 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085244003 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs/61c2066a8a0f%2C41921%2C1732085228346.1732085244003
2024-11-20T06:47:30,466 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 is not closed yet, will try archiving it next time
2024-11-20T06:47:30,854 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@265fafc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741863_1046 to 127.0.0.1:33321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:30,854 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d44a2d7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741868_1051 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:31,044 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T06:47:31,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/bf36533da7ba4e14a53241e50a902355 is 1079, key is tmprow/info:/1732085251042/Put/seqid=0
2024-11-20T06:47:31,054 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,054 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:31,055 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741879_1062
2024-11-20T06:47:31,055 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:31,056 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,057 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:31,057 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741880_1063
2024-11-20T06:47:31,057 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:31,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42755
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42318 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064 to mirror 127.0.0.1:42755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:31,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064
2024-11-20T06:47:31,060 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42318 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:31,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42318 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42318 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:31,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:31,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741882_1065
2024-11-20T06:47:31,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:31,063 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:31,063 WARN  [IPC Server handler 0 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:31,063 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:31,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741883_1066 (size=6027)
2024-11-20T06:47:31,230 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,468 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/bf36533da7ba4e14a53241e50a902355
2024-11-20T06:47:31,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/bf36533da7ba4e14a53241e50a902355 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355
2024-11-20T06:47:31,487 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355, entries=1, sequenceid=55, filesize=5.9 K
2024-11-20T06:47:31,488 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 7f30a91b3b12ffdaf1928b202e5ace79 in 443ms, sequenceid=55, compaction requested=true
2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b because midkey is the same as first or last row
2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f30a91b3b12ffdaf1928b202e5ace79:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:47:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:31,488 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:47:31,489 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:47:31,489 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1541): 7f30a91b3b12ffdaf1928b202e5ace79/info is initiating minor compaction (all files)
2024-11-20T06:47:31,489 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f30a91b3b12ffdaf1928b202e5ace79/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:31,490 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355] into tmpdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp, totalSize=29.3 K
2024-11-20T06:47:31,490 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84756580f4494a84b2f807cb98112c8b, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732085242050
2024-11-20T06:47:31,490 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd9e014241444abb1ccb0842577cedb, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732085249610
2024-11-20T06:47:31,491 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf36533da7ba4e14a53241e50a902355, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732085251042
2024-11-20T06:47:31,503 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f30a91b3b12ffdaf1928b202e5ace79#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:47:31,504 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/0264c63e18fc4c5e8170f1687de5d421 is 1080, key is row0002/info:/1732085242050/Put/seqid=0
2024-11-20T06:47:31,507 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42755
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42332 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067 to mirror 127.0.0.1:42755
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,507 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]) is bad.
2024-11-20T06:47:31,507 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067
2024-11-20T06:47:31,507 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42332 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:31,507 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42332 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42332 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,507 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42755,DS-b7eed287-7d6a-44f6-aad2-cb955120661c,DISK]
2024-11-20T06:47:31,508 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,509 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]) is bad.
2024-11-20T06:47:31,509 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741885_1068
2024-11-20T06:47:31,509 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]
2024-11-20T06:47:31,511 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42348 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,511 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:31,511 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069
2024-11-20T06:47:31,511 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42348 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:31,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42348 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42348 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,512 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:31,514 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39949
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,514 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42354 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070 to mirror 127.0.0.1:39949
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,514 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]) is bad.
2024-11-20T06:47:31,514 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070
2024-11-20T06:47:31,514 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42354 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:31,514 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:42354 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741887_1070] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42354 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:31,515 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39949,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]
2024-11-20T06:47:31,515 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-20T06:47:31,516 WARN  [IPC Server handler 0 on default port 45393 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-20T06:47:31,516 WARN  [IPC Server handler 0 on default port 45393 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-20T06:47:31,519 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741888_1071 (size=18097)
2024-11-20T06:47:31,587 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,587 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:31,929 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/0264c63e18fc4c5e8170f1687de5d421 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421
2024-11-20T06:47:31,938 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f30a91b3b12ffdaf1928b202e5ace79/info of 7f30a91b3b12ffdaf1928b202e5ace79 into 0264c63e18fc4c5e8170f1687de5d421(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:31,938 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79., storeName=7f30a91b3b12ffdaf1928b202e5ace79/info, priority=13, startTime=1732085251488; duration=0sec
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 because midkey is the same as first or last row
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:31,938 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 because midkey is the same as first or last row
2024-11-20T06:47:31,939 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-20T06:47:31,939 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:31,939 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 because midkey is the same as first or last row
2024-11-20T06:47:31,939 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:31,939 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f30a91b3b12ffdaf1928b202e5ace79:info
2024-11-20T06:47:32,066 WARN  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-20T06:47:32,285 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:32,289 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:32,292 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:32,292 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:32,292 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:47:32,292 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a8eeeb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:32,292 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6842affb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:32,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25f6ac6c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/java.io.tmpdir/jetty-localhost-40581-hadoop-hdfs-3_4_1-tests_jar-_-any-6400630024880205130/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:32,389 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2586080f{HTTP/1.1, (http/1.1)}{localhost:40581}
2024-11-20T06:47:32,389 INFO  [Time-limited test {}] server.Server(415): Started @135731ms
2024-11-20T06:47:32,390 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:32,760 WARN  [Thread-990 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:32,768 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9c398d1ace39f03 with lease ID 0xd87bea97461ac237: from storage DS-0356e157-8ecb-4880-a294-8b712f127c1d node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=46871, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:32,768 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9c398d1ace39f03 with lease ID 0xd87bea97461ac237: from storage DS-245946a7-a5ae-4fe8-89b6-6a61b564def7 node DatanodeRegistration(127.0.0.1:38303, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=46871, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:32,854 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@265fafc[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741858_1041 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:32,854 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d44a2d7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741873_1056 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:33,230 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:33,588 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:33,588 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:33,853 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3d44a2d7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45705, datanodeUuid=29de8e53-d66e-4fc4-9f86-418ea39c89fe, infoPort=46139, infoSecurePort=0, ipcPort=35489, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741883_1066 to 127.0.0.1:33321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:33,857 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741888_1071 (size=18097)
2024-11-20T06:47:35,231 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:35,589 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:35,589 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:36,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:47:37,232 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,505 ERROR [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData-prefix:61c2066a8a0f,38687,1732085226879 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,505 WARN  [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData-prefix:61c2066a8a0f,38687,1732085226879 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,506 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C38687%2C1732085226879:(num 1732085227258) roll requested
2024-11-20T06:47:37,507 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C38687%2C1732085226879.1732085257506
2024-11-20T06:47:37,517 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:37,517 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:37,517 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:37,517 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:37,517 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:37,517 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085257506
2024-11-20T06:47:37,518 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,518 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,518 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258
2024-11-20T06:47:37,518 WARN  [IPC Server handler 0 on default port 45393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-11-20T06:47:37,519 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 after 1ms
2024-11-20T06:47:37,521 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46871:46871),(127.0.0.1/127.0.0.1:46139:46139)]
2024-11-20T06:47:37,521 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 is not closed yet, will try archiving it next time
2024-11-20T06:47:37,589 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:37,589 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:39,590 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:39,590 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:41,520 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/WALs/61c2066a8a0f,38687,1732085226879/61c2066a8a0f%2C38687%2C1732085226879.1732085227258 after 4002ms
2024-11-20T06:47:41,591 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:41,591 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:42,783 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@73456e8d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:37307,null,null])
java.net.ConnectException: Call From 61c2066a8a0f/172.17.0.2 to localhost:38657 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T06:47:42,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741837_1020 (size=455)
2024-11-20T06:47:43,042 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085228575 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs/61c2066a8a0f%2C41921%2C1732085228346.1732085228575
2024-11-20T06:47:43,045 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085248027 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs/61c2066a8a0f%2C41921%2C1732085228346.1732085248027
2024-11-20T06:47:43,592 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:43,592 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:44,766 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42fcbfb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38303, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=46871, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741837_1020 to 127.0.0.1:42755 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:45,592 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:45,593 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:45,764 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41921%2C1732085228346.1732085265764
2024-11-20T06:47:45,773 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:45,773 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:45,773 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:45,773 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:45,774 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:45,774 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085250047 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085265764
2024-11-20T06:47:45,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741878_1061 (size=12911)
2024-11-20T06:47:45,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46139:46139),(127.0.0.1/127.0.0.1:46871:46871)]
2024-11-20T06:47:45,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085250047 is not closed yet, will try archiving it next time
2024-11-20T06:47:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:45,786 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-20T06:47:45,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/e92d00a51c4142d582f6785e62c92c3c is 1080, key is row0013/info:/1732085265783/Put/seqid=0
2024-11-20T06:47:45,794 WARN  [Thread-1027 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1075
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:45,795 WARN  [Thread-1027 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741891_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:45,795 WARN  [Thread-1027 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741891_1075
2024-11-20T06:47:45,796 WARN  [Thread-1027 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:45,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741892_1076 (size=8190)
2024-11-20T06:47:45,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741892_1076 (size=8190)
2024-11-20T06:47:45,814 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/e92d00a51c4142d582f6785e62c92c3c
2024-11-20T06:47:45,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/e92d00a51c4142d582f6785e62c92c3c as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c
2024-11-20T06:47:45,827 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c, entries=3, sequenceid=66, filesize=8.0 K
2024-11-20T06:47:45,828 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 7f30a91b3b12ffdaf1928b202e5ace79 in 42ms, sequenceid=66, compaction requested=false
2024-11-20T06:47:45,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:45,829 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-20T06:47:45,829 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:45,829 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 because midkey is the same as first or last row
2024-11-20T06:47:46,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41921 {}] regionserver.HRegion(8855): Flush requested on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:46,012 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7f30a91b3b12ffdaf1928b202e5ace79 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-20T06:47:46,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/150625e4e66c4545b5d1e6c8403961d7 is 1080, key is row0015/info:/1732085265787/Put/seqid=0
2024-11-20T06:47:46,020 WARN  [Thread-1034 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56242 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:46,021 WARN  [Thread-1034 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:46,021 WARN  [Thread-1034 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077
2024-11-20T06:47:46,021 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56242 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:46,021 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56242 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56242 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:46,021 WARN  [Thread-1034 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:46,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741894_1078 (size=14660)
2024-11-20T06:47:46,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741894_1078 (size=14660)
2024-11-20T06:47:46,026 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/150625e4e66c4545b5d1e6c8403961d7
2024-11-20T06:47:46,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/150625e4e66c4545b5d1e6c8403961d7 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7
2024-11-20T06:47:46,040 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7, entries=9, sequenceid=79, filesize=14.3 K
2024-11-20T06:47:46,041 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 7f30a91b3b12ffdaf1928b202e5ace79 in 29ms, sequenceid=79, compaction requested=true
2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 because midkey is the same as first or last row
2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7f30a91b3b12ffdaf1928b202e5ace79:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:47:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:46,041 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:47:46,042 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:47:46,043 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1541): 7f30a91b3b12ffdaf1928b202e5ace79/info is initiating minor compaction (all files)
2024-11-20T06:47:46,043 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7f30a91b3b12ffdaf1928b202e5ace79/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,043 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7] into tmpdir=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp, totalSize=40.0 K
2024-11-20T06:47:46,043 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0264c63e18fc4c5e8170f1687de5d421, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732085242050
2024-11-20T06:47:46,044 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting e92d00a51c4142d582f6785e62c92c3c, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732085252066
2024-11-20T06:47:46,044 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150625e4e66c4545b5d1e6c8403961d7, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732085265787
2024-11-20T06:47:46,056 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7f30a91b3b12ffdaf1928b202e5ace79#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:47:46,057 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/2aaaf83360524a2a8a9e7c2f7d6b70e3 is 1080, key is row0002/info:/1732085242050/Put/seqid=0
2024-11-20T06:47:46,059 WARN  [Thread-1043 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56266 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8]'}, localName='127.0.0.1:45705', datanodeUuid='29de8e53-d66e-4fc4-9f86-418ea39c89fe', xmitsInProgress=0}:Exception transferring block BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079 to mirror 127.0.0.1:33321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:46,060 WARN  [Thread-1043 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45705,DS-69db36d5-f018-443a-8c9c-853b6bf56aec,DISK], DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:46,060 WARN  [Thread-1043 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079
2024-11-20T06:47:46,060 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56266 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079] {}] datanode.BlockReceiver(316): Block 1073741895 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-20T06:47:46,060 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_508146164_22 at /127.0.0.1:56266 [Receiving block BP-127923120-172.17.0.2-1732085225158:blk_1073741895_1079] {}] datanode.DataXceiver(331): 127.0.0.1:45705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56266 dst: /127.0.0.1:45705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:46,060 WARN  [Thread-1043 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:46,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741896_1080 (size=28989)
2024-11-20T06:47:46,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741896_1080 (size=28989)
2024-11-20T06:47:46,068 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-20T06:47:46,072 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/.tmp/info/2aaaf83360524a2a8a9e7c2f7d6b70e3 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/2aaaf83360524a2a8a9e7c2f7d6b70e3
2024-11-20T06:47:46,080 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7f30a91b3b12ffdaf1928b202e5ace79/info of 7f30a91b3b12ffdaf1928b202e5ace79 into 2aaaf83360524a2a8a9e7c2f7d6b70e3(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7f30a91b3b12ffdaf1928b202e5ace79:

2024-11-20T06:47:46,080 INFO  [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79., storeName=7f30a91b3b12ffdaf1928b202e5ace79/info, priority=13, startTime=1732085266041; duration=0sec
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/2aaaf83360524a2a8a9e7c2f7d6b70e3 because midkey is the same as first or last row
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/2aaaf83360524a2a8a9e7c2f7d6b70e3 because midkey is the same as first or last row
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/2aaaf83360524a2a8a9e7c2f7d6b70e3 because midkey is the same as first or last row
2024-11-20T06:47:46,080 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:47:46,081 DEBUG [RS:1;61c2066a8a0f:41921-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7f30a91b3b12ffdaf1928b202e5ace79:info
2024-11-20T06:47:46,177 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41921,1732085228346/61c2066a8a0f%2C41921%2C1732085228346.1732085250047 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs/61c2066a8a0f%2C41921%2C1732085228346.1732085250047
2024-11-20T06:47:46,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:47:46,213 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:47:46,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:46,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,213 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:47:46,214 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:47:46,214 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=318977838, stopped=false
2024-11-20T06:47:46,214 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,38687,1732085226879
2024-11-20T06:47:46,263 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:46,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:46,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:47:46,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:46,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:46,263 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:46,263 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:47:46,264 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:47:46,264 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:46,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,41993,1732085227061' *****
2024-11-20T06:47:46,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:46,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:46,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:47:46,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,41921,1732085228346' *****
2024-11-20T06:47:46,264 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:47:46,265 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:47:46,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:47:46,265 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(3091): Received CLOSE for 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:47:46,265 INFO  [RS:0;61c2066a8a0f:41993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:41993.
2024-11-20T06:47:46,265 DEBUG [RS:0;61c2066a8a0f:41993 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:46,265 DEBUG [RS:0;61c2066a8a0f:41993 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:46,265 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:47:46,266 INFO  [RS:1;61c2066a8a0f:41921 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;61c2066a8a0f:41921.
2024-11-20T06:47:46,266 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:47:46,266 DEBUG [RS:1;61c2066a8a0f:41921 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:47:46,266 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7f30a91b3b12ffdaf1928b202e5ace79, disabling compactions & flushes
2024-11-20T06:47:46,266 DEBUG [RS:1;61c2066a8a0f:41921 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,266 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:47:46,266 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,266 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,266 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79. after waiting 0 ms
2024-11-20T06:47:46,266 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1325): Online Regions={7f30a91b3b12ffdaf1928b202e5ace79=TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.}
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,266 DEBUG [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1351): Waiting on 7f30a91b3b12ffdaf1928b202e5ace79
2024-11-20T06:47:46,266 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T06:47:46,266 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:47:46,266 DEBUG [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:47:46,266 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:47:46,266 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:47:46,266 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7] to archive
2024-11-20T06:47:46,267 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-20T06:47:46,267 ERROR [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a-prefix:61c2066a8a0f,41993,1732085227061.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,267 WARN  [FSHLog-0-hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a-prefix:61c2066a8a0f,41993,1732085227061.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,267 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C41993%2C1732085227061.meta:.meta(num 1732085228142) roll requested
2024-11-20T06:47:46,268 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41993%2C1732085227061.meta.1732085266268.meta
2024-11-20T06:47:46,268 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:47:46,270 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/c3b0002228e0486aac32db3e32251ab2
2024-11-20T06:47:46,272 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/b9051486efe0471d8caddd6a1fb82e11
2024-11-20T06:47:46,273 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/84756580f4494a84b2f807cb98112c8b
2024-11-20T06:47:46,275 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/463e5f1646f644fc8c99480777208f95
2024-11-20T06:47:46,275 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,275 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,275 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,276 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,276 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,276 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085266268.meta
2024-11-20T06:47:46,276 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,276 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/8cd9e014241444abb1ccb0842577cedb
2024-11-20T06:47:46,276 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,276 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
2024-11-20T06:47:46,277 WARN  [IPC Server handler 0 on default port 45393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta has not been closed. Lease recovery is in progress. RecoveryId = 1082 for block blk_1073741834_1010
2024-11-20T06:47:46,277 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta after 1ms
2024-11-20T06:47:46,277 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/0264c63e18fc4c5e8170f1687de5d421
2024-11-20T06:47:46,279 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/bf36533da7ba4e14a53241e50a902355
2024-11-20T06:47:46,280 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/e92d00a51c4142d582f6785e62c92c3c
2024-11-20T06:47:46,282 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7 to hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/info/150625e4e66c4545b5d1e6c8403961d7
2024-11-20T06:47:46,282 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=61c2066a8a0f:38687 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-20T06:47:46,282 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c3b0002228e0486aac32db3e32251ab2=10347, b9051486efe0471d8caddd6a1fb82e11=12506, 84756580f4494a84b2f807cb98112c8b=17994, 463e5f1646f644fc8c99480777208f95=6027, 8cd9e014241444abb1ccb0842577cedb=6027, 0264c63e18fc4c5e8170f1687de5d421=18097, bf36533da7ba4e14a53241e50a902355=6027, e92d00a51c4142d582f6785e62c92c3c=8190, 150625e4e66c4545b5d1e6c8403961d7=14660]
2024-11-20T06:47:46,282 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46139:46139),(127.0.0.1/127.0.0.1:46871:46871)]
2024-11-20T06:47:46,283 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta is not closed yet, will try archiving it next time
2024-11-20T06:47:46,291 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/default/TestLogRolling-testLogRollOnDatanodeDeath/7f30a91b3b12ffdaf1928b202e5ace79/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1
2024-11-20T06:47:46,292 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,292 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7f30a91b3b12ffdaf1928b202e5ace79:
Waiting for close lock at 1732085266266Running coprocessor pre-close hooks at 1732085266266Disabling compacts and flushes for region at 1732085266266Disabling writes for close at 1732085266266Writing region close event to WAL at 1732085266283 (+17 ms)Running coprocessor post-close hooks at 1732085266291 (+8 ms)Closed at 1732085266292 (+1 ms)
2024-11-20T06:47:46,292 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79.
2024-11-20T06:47:46,300 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/info/31cc0b2ba51d4702b573b69ce46dc509 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732085228472.7f30a91b3b12ffdaf1928b202e5ace79./info:regioninfo/1732085229248/Put/seqid=0
2024-11-20T06:47:46,302 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,302 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741898_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:38303,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:46,302 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741898_1083
2024-11-20T06:47:46,303 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:46,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741899_1084 (size=7089)
2024-11-20T06:47:46,312 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741899_1084 (size=7089)
2024-11-20T06:47:46,313 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/info/31cc0b2ba51d4702b573b69ce46dc509
2024-11-20T06:47:46,334 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/ns/75dac2c19f4548b791d063281233f43c is 43, key is default/ns:d/1732085228262/Put/seqid=0
2024-11-20T06:47:46,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741900_1085 (size=5153)
2024-11-20T06:47:46,339 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741900_1085 (size=5153)
2024-11-20T06:47:46,339 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/ns/75dac2c19f4548b791d063281233f43c
2024-11-20T06:47:46,359 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/table/93a995661da24dbdb67e35feacecdfaf is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732085229263/Put/seqid=0
2024-11-20T06:47:46,361 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:46,361 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-127923120-172.17.0.2-1732085225158:blk_1073741901_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK], DatanodeInfoWithStorage[127.0.0.1:38303,DS-0356e157-8ecb-4880-a294-8b712f127c1d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]) is bad.
2024-11-20T06:47:46,361 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-127923120-172.17.0.2-1732085225158:blk_1073741901_1086
2024-11-20T06:47:46,361 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33321,DS-c5bd5667-106d-48f6-a87d-cd57f52cbd94,DISK]
2024-11-20T06:47:46,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741902_1087 (size=5424)
2024-11-20T06:47:46,365 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741902_1087 (size=5424)
2024-11-20T06:47:46,366 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/table/93a995661da24dbdb67e35feacecdfaf
2024-11-20T06:47:46,371 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/info/31cc0b2ba51d4702b573b69ce46dc509 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/info/31cc0b2ba51d4702b573b69ce46dc509
2024-11-20T06:47:46,378 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/info/31cc0b2ba51d4702b573b69ce46dc509, entries=10, sequenceid=11, filesize=6.9 K
2024-11-20T06:47:46,379 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/ns/75dac2c19f4548b791d063281233f43c as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/ns/75dac2c19f4548b791d063281233f43c
2024-11-20T06:47:46,385 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/ns/75dac2c19f4548b791d063281233f43c, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T06:47:46,387 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/.tmp/table/93a995661da24dbdb67e35feacecdfaf as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/table/93a995661da24dbdb67e35feacecdfaf
2024-11-20T06:47:46,393 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/table/93a995661da24dbdb67e35feacecdfaf, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T06:47:46,395 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 129ms, sequenceid=11, compaction requested=false
2024-11-20T06:47:46,400 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T06:47:46,401 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:47:46,401 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:46,401 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085266266Running coprocessor pre-close hooks at 1732085266266Disabling compacts and flushes for region at 1732085266266Disabling writes for close at 1732085266266Obtaining lock to block concurrent updates at 1732085266267 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732085266267Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732085266267Flushing stores of hbase:meta,,1.1588230740 at 1732085266284 (+17 ms)Flushing 1588230740/info: creating writer at 1732085266284Flushing 1588230740/info: appending metadata at 1732085266299 (+15 ms)Flushing 1588230740/info: closing flushed file at 1732085266299Flushing 1588230740/ns: creating writer at 1732085266320 (+21 ms)Flushing 1588230740/ns: appending metadata at 1732085266333 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732085266333Flushing 1588230740/table: creating writer at 1732085266345 (+12 ms)Flushing 1588230740/table: appending metadata at 1732085266358 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732085266358Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ddb5cc1: reopening flushed file at 1732085266370 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6adb6fcf: reopening flushed file at 1732085266379 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e6b5080: reopening flushed file at 1732085266385 (+6 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 129ms, sequenceid=11, compaction requested=false at 1732085266395 (+10 ms)Writing region close event to WAL at 1732085266396 (+1 ms)Running coprocessor post-close hooks at 1732085266401 (+5 ms)Closed at 1732085266401
2024-11-20T06:47:46,401 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:47:46,441 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T06:47:46,441 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T06:47:46,443 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:46,466 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,41921,1732085228346; all regions closed.
2024-11-20T06:47:46,466 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,41993,1732085227061; all regions closed.
2024-11-20T06:47:46,467 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,467 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741890_1074 (size=16308)
2024-11-20T06:47:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741890_1074 (size=16308)
2024-11-20T06:47:46,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741897_1081 (size=825)
2024-11-20T06:47:46,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741897_1081 (size=825)
2024-11-20T06:47:46,472 DEBUG [RS:1;61c2066a8a0f:41921 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C41921%2C1732085228346:(num 1732085265764)
2024-11-20T06:47:46,472 DEBUG [RS:1;61c2066a8a0f:41921 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:47:46,472 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:47:46,472 INFO  [RS:1;61c2066a8a0f:41921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41921
2024-11-20T06:47:46,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:46,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,41921,1732085228346
2024-11-20T06:47:46,497 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:46,505 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,41921,1732085228346]
2024-11-20T06:47:46,513 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,41921,1732085228346 already deleted, retry=false
2024-11-20T06:47:46,513 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,41921,1732085228346 expired; onlineServers=1
2024-11-20T06:47:46,587 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T06:47:46,587 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T06:47:46,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:46,605 INFO  [RS:1;61c2066a8a0f:41921 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:46,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41921-0x1015848b02d0002, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:46,605 INFO  [RS:1;61c2066a8a0f:41921 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,41921,1732085228346; zookeeper connection closed.
2024-11-20T06:47:46,606 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75ccaff0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75ccaff0
2024-11-20T06:47:47,589 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:47,769 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@42fcbfb9[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38303, datanodeUuid=d67726cb-0b7c-4be9-93ad-8ddfa3123cc6, infoPort=46871, infoSecurePort=0, ipcPort=36487, storageInfo=lv=-57;cid=testClusterID;nsid=249829610;c=1732085225158):Failed to transfer BP-127923120-172.17.0.2-1732085225158:blk_1073741835_1011 to 127.0.0.1:33321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:47,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:47:48,283 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:48,291 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T06:47:48,291 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T06:47:48,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:47:48,768 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:47:50,279 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta after 4003ms
2024-11-20T06:47:50,283 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:50,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:47:50,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:47:50,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741878_1061 (size=12911)
2024-11-20T06:47:51,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,320 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,321 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,322 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,468 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-20T06:47:51,476 DEBUG [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs
2024-11-20T06:47:51,477 INFO  [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C41993%2C1732085227061.meta:.meta(num 1732085266268)
2024-11-20T06:47:51,478 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:51,478 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:51,478 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:51,478 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:51,479 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:51,480 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:51,480 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:37307,DS-e12a774d-9445-4470-8205-7833e690accd,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:47:51,480 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
2024-11-20T06:47:51,481 WARN  [IPC Server handler 0 on default port 45393 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 has not been closed. Lease recovery is in progress. RecoveryId = 1088 for block blk_1073741833_1009
2024-11-20T06:47:51,482 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 after 1ms
2024-11-20T06:47:51,837 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:47:51,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,860 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,865 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:51,866 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:52,284 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-20T06:47:52,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:52,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:47:52,787 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@33d030a1 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:37307,null,null])
java.net.ConnectException: Call From 61c2066a8a0f/172.17.0.2 to localhost:38657 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T06:47:53,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-20T06:47:53,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:47:53,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:47:53,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:53,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741830_1073 (size=27306)
2024-11-20T06:47:54,284 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-20T06:47:54,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:55,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:55,483 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 after 4003ms
2024-11-20T06:47:56,285 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-20T06:47:56,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:56,480 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-20T06:47:56,482 DEBUG [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/oldWALs
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C41993%2C1732085227061:(num 1732085227723)
2024-11-20T06:47:56,482 DEBUG [RS:0;61c2066a8a0f:41993 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:47:56,482 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:47:56,482 INFO  [RS:0;61c2066a8a0f:41993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41993
2024-11-20T06:47:56,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:56,545 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,41993,1732085227061
2024-11-20T06:47:56,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:47:56,546 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:56,546 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,41993,1732085227061]
2024-11-20T06:47:56,562 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,41993,1732085227061 already deleted, retry=false
2024-11-20T06:47:56,562 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,41993,1732085227061 expired; onlineServers=0
2024-11-20T06:47:56,562 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,38687,1732085226879' *****
2024-11-20T06:47:56,562 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:47:56,562 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:47:56,563 DEBUG [M:0;61c2066a8a0f:38687 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:47:56,563 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:47:56,563 DEBUG [M:0;61c2066a8a0f:38687 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:47:56,563 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085227508 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085227508,5,FailOnTimeoutGroup]
2024-11-20T06:47:56,563 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085227508 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085227508,5,FailOnTimeoutGroup]
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:47:56,563 DEBUG [M:0;61c2066a8a0f:38687 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:47:56,563 INFO  [M:0;61c2066a8a0f:38687 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:47:56,564 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:47:56,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:47:56,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:47:56,571 DEBUG [M:0;61c2066a8a0f:38687 {}] zookeeper.ZKUtil(347): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:47:56,571 WARN  [M:0;61c2066a8a0f:38687 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:47:56,571 INFO  [M:0;61c2066a8a0f:38687 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/.lastflushedseqids
2024-11-20T06:47:56,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741903_1089 (size=130)
2024-11-20T06:47:56,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741903_1089 (size=130)
2024-11-20T06:47:56,579 INFO  [M:0;61c2066a8a0f:38687 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:47:56,579 INFO  [M:0;61c2066a8a0f:38687 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:47:56,579 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:47:56,579 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:56,579 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:56,579 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:47:56,579 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:56,580 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-20T06:47:56,603 DEBUG [M:0;61c2066a8a0f:38687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacad74fa8424fa087f563258bf7e344 is 82, key is hbase:meta,,1/info:regioninfo/1732085228175/Put/seqid=0
2024-11-20T06:47:56,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741904_1090 (size=5672)
2024-11-20T06:47:56,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741904_1090 (size=5672)
2024-11-20T06:47:56,654 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:56,654 INFO  [RS:0;61c2066a8a0f:41993 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:56,654 INFO  [RS:0;61c2066a8a0f:41993 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,41993,1732085227061; zookeeper connection closed.
2024-11-20T06:47:56,654 DEBUG [pool-302-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41993-0x1015848b02d0001, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:56,654 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@202db4f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@202db4f
2024-11-20T06:47:56,654 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-20T06:47:56,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741836_1012 (size=76)
2024-11-20T06:47:56,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:57,010 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacad74fa8424fa087f563258bf7e344
2024-11-20T06:47:57,047 DEBUG [M:0;61c2066a8a0f:38687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b847213123234543bff878a2a317a559 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085229271/Put/seqid=0
2024-11-20T06:47:57,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741905_1091 (size=6255)
2024-11-20T06:47:57,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741905_1091 (size=6255)
2024-11-20T06:47:57,057 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b847213123234543bff878a2a317a559
2024-11-20T06:47:57,063 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b847213123234543bff878a2a317a559
2024-11-20T06:47:57,080 DEBUG [M:0;61c2066a8a0f:38687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9103f76fd5d9441e88c4f94e91f175eb is 69, key is 61c2066a8a0f,41921,1732085228346/rs:state/1732085228421/Put/seqid=0
2024-11-20T06:47:57,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741906_1092 (size=5224)
2024-11-20T06:47:57,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741906_1092 (size=5224)
2024-11-20T06:47:57,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:57,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:57,492 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9103f76fd5d9441e88c4f94e91f175eb
2024-11-20T06:47:57,531 DEBUG [M:0;61c2066a8a0f:38687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b48407921bb64aa784f154fffcc45456 is 52, key is load_balancer_on/state:d/1732085228330/Put/seqid=0
2024-11-20T06:47:57,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741907_1093 (size=5056)
2024-11-20T06:47:57,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741907_1093 (size=5056)
2024-11-20T06:47:57,541 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b48407921bb64aa784f154fffcc45456
2024-11-20T06:47:57,547 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eacad74fa8424fa087f563258bf7e344 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eacad74fa8424fa087f563258bf7e344
2024-11-20T06:47:57,555 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/eacad74fa8424fa087f563258bf7e344, entries=8, sequenceid=60, filesize=5.5 K
2024-11-20T06:47:57,556 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b847213123234543bff878a2a317a559 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b847213123234543bff878a2a317a559
2024-11-20T06:47:57,562 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b847213123234543bff878a2a317a559
2024-11-20T06:47:57,563 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b847213123234543bff878a2a317a559, entries=6, sequenceid=60, filesize=6.1 K
2024-11-20T06:47:57,564 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9103f76fd5d9441e88c4f94e91f175eb as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9103f76fd5d9441e88c4f94e91f175eb
2024-11-20T06:47:57,570 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9103f76fd5d9441e88c4f94e91f175eb, entries=2, sequenceid=60, filesize=5.1 K
2024-11-20T06:47:57,572 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b48407921bb64aa784f154fffcc45456 as hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b48407921bb64aa784f154fffcc45456
2024-11-20T06:47:57,579 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b48407921bb64aa784f154fffcc45456, entries=1, sequenceid=60, filesize=4.9 K
2024-11-20T06:47:57,580 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1001ms, sequenceid=60, compaction requested=false
2024-11-20T06:47:57,582 INFO  [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:47:57,583 DEBUG [M:0;61c2066a8a0f:38687 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085276579Disabling compacts and flushes for region at 1732085276579Disabling writes for close at 1732085276579Obtaining lock to block concurrent updates at 1732085276580 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085276580Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1732085276580Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085276581 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085276581Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085276602 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085276602Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085277026 (+424 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085277046 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085277047 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085277063 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085277079 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085277079Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085277505 (+426 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085277531 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085277531Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79cca358: reopening flushed file at 1732085277546 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d250b1a: reopening flushed file at 1732085277555 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@685617a1: reopening flushed file at 1732085277563 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e396766: reopening flushed file at 1732085277571 (+8 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1001ms, sequenceid=60, compaction requested=false at 1732085277580 (+9 ms)Writing region close event to WAL at 1732085277582 (+2 ms)Closed at 1732085277582
2024-11-20T06:47:57,583 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:57,584 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:57,584 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:57,584 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:57,584 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:47:57,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38303 is added to blk_1073741889_1072 (size=1045)
2024-11-20T06:47:57,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45705 is added to blk_1073741889_1072 (size=1045)
2024-11-20T06:47:57,587 INFO  [M:0;61c2066a8a0f:38687 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:47:57,587 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:47:57,588 INFO  [M:0;61c2066a8a0f:38687 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38687
2024-11-20T06:47:57,588 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:47:57,745 INFO  [M:0;61c2066a8a0f:38687 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:47:57,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:57,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38687-0x1015848b02d0000, quorum=127.0.0.1:62382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:47:57,748 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25f6ac6c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:57,749 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2586080f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:57,749 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:57,749 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6842affb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:57,749 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a8eeeb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:57,751 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:57,751 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:57,751 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid d67726cb-0b7c-4be9-93ad-8ddfa3123cc6) service to localhost/127.0.0.1:45393
2024-11-20T06:47:57,751 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:57,752 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4b83bf39 {}] ipc.Client$Connection(956): Interrupted while trying for connection
2024-11-20T06:47:57,751 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@218dfe36 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:37307,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:38657 , LocalHost:localPort 61c2066a8a0f/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T06:47:57,752 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data3/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:57,752 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@218dfe36 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:38303,null,null])
java.io.IOException: No block pool offer service for bpid=BP-127923120-172.17.0.2-1732085225158
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:57,752 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4b83bf39 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37307,null,null])
java.net.ConnectException: Call From 61c2066a8a0f/172.17.0.2 to localhost:38657 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-20T06:47:57,753 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@218dfe36 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37307,null,null], DatanodeInfoWithStorage[127.0.0.1:38303,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-127923120-172.17.0.2-1732085225158:blk_1073741833_1009, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37307,null,null], DatanodeInfoWithStorage[127.0.0.1:38303,null,null]]
2024-11-20T06:47:57,753 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4b83bf39 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38303,null,null])
java.io.IOException: No block pool offer service for bpid=BP-127923120-172.17.0.2-1732085225158
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:47:57,753 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4b83bf39 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:37307,null,null], DatanodeInfoWithStorage[127.0.0.1:38303,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-127923120-172.17.0.2-1732085225158:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:37307,null,null], DatanodeInfoWithStorage[127.0.0.1:38303,null,null]]
2024-11-20T06:47:57,753 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data4/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:57,753 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:57,756 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70a0068f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:57,757 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39615ad8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:57,757 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:57,757 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bea65f7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:57,758 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3eb2cdd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:57,760 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:47:57,760 WARN  [BP-127923120-172.17.0.2-1732085225158 heartbeating to localhost/127.0.0.1:45393 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-127923120-172.17.0.2-1732085225158 (Datanode Uuid 29de8e53-d66e-4fc4-9f86-418ea39c89fe) service to localhost/127.0.0.1:45393
2024-11-20T06:47:57,760 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:47:57,760 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:47:57,761 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data7/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:57,761 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/cluster_9a19091d-2da1-ca71-2772-e33186a81053/data/data8/current/BP-127923120-172.17.0.2-1732085225158 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:47:57,761 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:47:57,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7982676d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:57,768 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2efbdc75{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:47:57,768 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:47:57,768 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d790455{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:47:57,768 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3150e6db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir/,STOPPED}
2024-11-20T06:47:57,776 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:47:57,820 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:47:57,829 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=155 (was 78)
Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42115
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fabc4bf44c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45393 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:42115
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:45393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45393
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007fabc4bf44c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:45393
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45393 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45393 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=431 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 232) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4034 (was 4997)
2024-11-20T06:47:57,837 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=155, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=250, ProcessCount=11, AvailableMemoryMB=4034
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.log.dir so I do NOT create it in target/test-data/f73a0a54-e86e-c05f-933f-588e06621604
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5854d7d2-2a6f-7652-601e-48ad774b2605/hadoop.tmp.dir so I do NOT create it in target/test-data/f73a0a54-e86e-c05f-933f-588e06621604
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e, deleteOnExit=true
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/test.cache.data in system properties and HBase conf
2024-11-20T06:47:57,838 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:47:57,839 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:57,839 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:57,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:47:57,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:47:57,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:47:57,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:47:57,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:47:57,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:47:57,857 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:58,189 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:58,197 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:58,205 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:58,205 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:58,205 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:58,211 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:58,211 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7096145a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:58,211 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10c583a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:58,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:58,316 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@232fa1ae{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-35219-hadoop-hdfs-3_4_1-tests_jar-_-any-13572410576648298513/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:47:58,316 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c07fc0{HTTP/1.1, (http/1.1)}{localhost:35219}
2024-11-20T06:47:58,316 INFO  [Time-limited test {}] server.Server(415): Started @161658ms
2024-11-20T06:47:58,329 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:47:58,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:58,520 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:47:58,536 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,536 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,536 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,536 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,537 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,537 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,541 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,541 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,542 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,545 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:47:58,552 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:58,556 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:58,556 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:58,556 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:58,557 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:47:58,557 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13646a74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:58,558 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@301612e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:58,667 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44e11af1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-33857-hadoop-hdfs-3_4_1-tests_jar-_-any-13980265732190369380/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:58,667 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@60043bb8{HTTP/1.1, (http/1.1)}{localhost:33857}
2024-11-20T06:47:58,668 INFO  [Time-limited test {}] server.Server(415): Started @162010ms
2024-11-20T06:47:58,669 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:58,700 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:47:58,704 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:47:58,704 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:47:58,705 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:47:58,705 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:47:58,705 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47946b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:47:58,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23accf28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:47:58,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77c83f30{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-36899-hadoop-hdfs-3_4_1-tests_jar-_-any-14891742322550759737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:47:58,818 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54dbaae8{HTTP/1.1, (http/1.1)}{localhost:36899}
2024-11-20T06:47:58,819 INFO  [Time-limited test {}] server.Server(415): Started @162161ms
2024-11-20T06:47:58,820 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:47:59,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:59,394 WARN  [Thread-1214 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data2/current/BP-1488376784-172.17.0.2-1732085277870/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:59,394 WARN  [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data1/current/BP-1488376784-172.17.0.2-1732085277870/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:59,419 WARN  [Thread-1177 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:59,425 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c12e7458fc3f07 with lease ID 0x381b7ee14fd19907: Processing first storage report for DS-e2ff7347-8779-4a65-b08d-5c101395526b from datanode DatanodeRegistration(127.0.0.1:34783, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=45889, infoSecurePort=0, ipcPort=46009, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870)
2024-11-20T06:47:59,425 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c12e7458fc3f07 with lease ID 0x381b7ee14fd19907: from storage DS-e2ff7347-8779-4a65-b08d-5c101395526b node DatanodeRegistration(127.0.0.1:34783, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=45889, infoSecurePort=0, ipcPort=46009, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:59,425 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2c12e7458fc3f07 with lease ID 0x381b7ee14fd19907: Processing first storage report for DS-aeb9a505-6d1e-4abf-ace3-c8b3a351ed52 from datanode DatanodeRegistration(127.0.0.1:34783, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=45889, infoSecurePort=0, ipcPort=46009, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870)
2024-11-20T06:47:59,425 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2c12e7458fc3f07 with lease ID 0x381b7ee14fd19907: from storage DS-aeb9a505-6d1e-4abf-ace3-c8b3a351ed52 node DatanodeRegistration(127.0.0.1:34783, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=45889, infoSecurePort=0, ipcPort=46009, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:47:59,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:47:59,555 WARN  [Thread-1225 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data4/current/BP-1488376784-172.17.0.2-1732085277870/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:59,555 WARN  [Thread-1224 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data3/current/BP-1488376784-172.17.0.2-1732085277870/current, will proceed with Du for space computation calculation, 
2024-11-20T06:47:59,615 WARN  [Thread-1200 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:47:59,618 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96ed0dc51b955bd7 with lease ID 0x381b7ee14fd19908: Processing first storage report for DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8 from datanode DatanodeRegistration(127.0.0.1:43607, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=37367, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870)
2024-11-20T06:47:59,619 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96ed0dc51b955bd7 with lease ID 0x381b7ee14fd19908: from storage DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8 node DatanodeRegistration(127.0.0.1:43607, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=37367, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:59,619 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96ed0dc51b955bd7 with lease ID 0x381b7ee14fd19908: Processing first storage report for DS-94c52b76-c37a-4a58-93de-fa39d0ee8486 from datanode DatanodeRegistration(127.0.0.1:43607, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=37367, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870)
2024-11-20T06:47:59,619 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96ed0dc51b955bd7 with lease ID 0x381b7ee14fd19908: from storage DS-94c52b76-c37a-4a58-93de-fa39d0ee8486 node DatanodeRegistration(127.0.0.1:43607, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=37367, infoSecurePort=0, ipcPort=37083, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:47:59,678 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604
2024-11-20T06:47:59,681 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/zookeeper_0, clientPort=58486, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:47:59,682 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58486
2024-11-20T06:47:59,683 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,685 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:59,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:47:59,703 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7 with version=8
2024-11-20T06:47:59,703 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:47:59,706 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:59,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,707 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:59,707 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,707 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:59,707 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:47:59,707 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:59,712 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43353
2024-11-20T06:47:59,715 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43353 connecting to ZooKeeper ensemble=127.0.0.1:58486
2024-11-20T06:47:59,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433530x0, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:59,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43353-0x10158497e850000 connected
2024-11-20T06:47:59,845 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,847 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:59,850 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7, hbase.cluster.distributed=false
2024-11-20T06:47:59,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:59,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43353
2024-11-20T06:47:59,855 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43353
2024-11-20T06:47:59,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43353
2024-11-20T06:47:59,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43353
2024-11-20T06:47:59,871 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43353
2024-11-20T06:47:59,894 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:47:59,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,894 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:47:59,895 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:47:59,895 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:47:59,895 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:47:59,895 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:47:59,899 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42245
2024-11-20T06:47:59,901 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42245 connecting to ZooKeeper ensemble=127.0.0.1:58486
2024-11-20T06:47:59,903 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,906 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:47:59,951 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422450x0, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:47:59,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:47:59,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42245-0x10158497e850001 connected
2024-11-20T06:47:59,952 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:47:59,953 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:47:59,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:47:59,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:47:59,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42245
2024-11-20T06:47:59,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42245
2024-11-20T06:47:59,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42245
2024-11-20T06:47:59,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42245
2024-11-20T06:47:59,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42245
2024-11-20T06:47:59,979 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:43353
2024-11-20T06:47:59,983 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,43353,1732085279706
2024-11-20T06:47:59,993 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:59,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:47:59,993 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,003 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:48:00,003 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,004 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:48:00,004 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,43353,1732085279706 from backup master directory
2024-11-20T06:48:00,011 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:00,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,011 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:48:00,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:00,012 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,021 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/hbase.id] with ID: cb98945a-4b24-46e5-897b-4eaa0e2db404
2024-11-20T06:48:00,021 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/.tmp/hbase.id
2024-11-20T06:48:00,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:48:00,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:48:00,054 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/.tmp/hbase.id]:[hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/hbase.id]
2024-11-20T06:48:00,073 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:00,073 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:48:00,076 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-20T06:48:00,095 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:48:00,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:48:00,120 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:48:00,121 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:48:00,121 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:00,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:48:00,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:48:00,147 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store
2024-11-20T06:48:00,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:48:00,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:48:00,167 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:00,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:48:00,168 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:00,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:00,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:48:00,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:00,168 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:00,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085280167Disabling compacts and flushes for region at 1732085280167Disabling writes for close at 1732085280168 (+1 ms)Writing region close event to WAL at 1732085280168Closed at 1732085280168
2024-11-20T06:48:00,170 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/.initializing
2024-11-20T06:48:00,170 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,174 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C43353%2C1732085279706, suffix=, logDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706, archiveDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/oldWALs, maxLogs=10
2024-11-20T06:48:00,175 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C43353%2C1732085279706.1732085280174
2024-11-20T06:48:00,200 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174
2024-11-20T06:48:00,223 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45889:45889),(127.0.0.1/127.0.0.1:37367:37367)]
2024-11-20T06:48:00,231 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:00,231 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:00,231 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,231 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,238 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:00,240 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:48:00,240 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,241 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:00,242 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:00,244 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:48:00,244 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,245 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:00,246 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:00,247 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:48:00,248 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,248 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:00,248 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:00,250 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:48:00,250 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,251 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:00,251 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,252 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,253 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,254 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,255 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:48:00,257 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:00,275 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:00,276 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859365, jitterRate=0.09273947775363922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:48:00,277 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085280231Initializing all the Stores at 1732085280233 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085280233Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085280238 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085280238Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085280238Cleaning up temporary data from old regions at 1732085280254 (+16 ms)Region opened successfully at 1732085280277 (+23 ms)
2024-11-20T06:48:00,278 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:48:00,282 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7a3587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:48:00,284 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:48:00,284 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:48:00,284 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:48:00,284 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:48:00,285 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:48:00,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:48:00,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:48:00,294 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:48:00,295 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:48:00,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:00,320 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:48:00,321 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:48:00,322 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:48:00,334 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:48:00,336 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:48:00,348 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:48:00,353 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:48:00,354 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:48:00,361 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:48:00,364 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:48:00,370 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:48:00,378 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:00,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:00,378 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,382 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,43353,1732085279706, sessionid=0x10158497e850000, setting cluster-up flag (Was=false)
2024-11-20T06:48:00,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,403 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,428 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:48:00,429 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,445 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:00,470 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:48:00,471 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:00,473 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:48:00,475 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:00,475 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:48:00,476 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:48:00,476 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,43353,1732085279706
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:48:00,478 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,482 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085310482
2024-11-20T06:48:00,482 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:00,482 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:48:00,482 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:48:00,482 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:48:00,482 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:48:00,482 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:48:00,483 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:48:00,484 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,484 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085280484,5,FailOnTimeoutGroup]
2024-11-20T06:48:00,484 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085280484,5,FailOnTimeoutGroup]
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,484 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:48:00,484 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:00,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:48:00,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:48:00,509 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:48:00,510 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7
2024-11-20T06:48:00,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:48:00,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:48:00,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:00,534 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:48:00,536 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:48:00,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,537 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:00,537 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:48:00,539 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:48:00,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,539 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:00,540 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:48:00,541 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:48:00,541 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,542 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:00,542 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:48:00,544 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:48:00,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:00,545 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:00,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:48:00,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740
2024-11-20T06:48:00,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740
2024-11-20T06:48:00,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:48:00,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:48:00,548 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:48:00,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:48:00,552 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:00,553 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795627, jitterRate=0.011692807078361511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:48:00,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085280533Initializing all the Stores at 1732085280534 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085280534Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085280534Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085280534Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085280534Cleaning up temporary data from old regions at 1732085280548 (+14 ms)Region opened successfully at 1732085280553 (+5 ms)
2024-11-20T06:48:00,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:48:00,554 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:48:00,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:48:00,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:48:00,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:48:00,554 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:48:00,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085280554Disabling compacts and flushes for region at 1732085280554Disabling writes for close at 1732085280554Writing region close event to WAL at 1732085280554Closed at 1732085280554
2024-11-20T06:48:00,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:00,556 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:48:00,556 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:48:00,557 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:48:00,559 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:48:00,569 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(746): ClusterId : cb98945a-4b24-46e5-897b-4eaa0e2db404
2024-11-20T06:48:00,569 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:48:00,585 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:48:00,585 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:48:00,595 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:48:00,596 DEBUG [RS:0;61c2066a8a0f:42245 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcd94e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:48:00,610 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:42245
2024-11-20T06:48:00,610 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:48:00,610 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:48:00,610 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:48:00,611 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,43353,1732085279706 with port=42245, startcode=1732085279893
2024-11-20T06:48:00,611 DEBUG [RS:0;61c2066a8a0f:42245 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:48:00,614 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43587, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:48:00,614 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43353 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,614 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43353 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,616 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7
2024-11-20T06:48:00,616 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41921
2024-11-20T06:48:00,616 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:48:00,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:48:00,629 DEBUG [RS:0;61c2066a8a0f:42245 {}] zookeeper.ZKUtil(111): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,629 WARN  [RS:0;61c2066a8a0f:42245 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:48:00,629 INFO  [RS:0;61c2066a8a0f:42245 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:00,629 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,629 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,42245,1732085279893]
2024-11-20T06:48:00,635 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:48:00,638 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:48:00,639 INFO  [RS:0;61c2066a8a0f:42245 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:48:00,639 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,642 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:48:00,644 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:48:00,644 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:48:00,644 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:48:00,645 DEBUG [RS:0;61c2066a8a0f:42245 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:48:00,650 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,651 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,651 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,651 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,651 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,651 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42245,1732085279893-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:48:00,670 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:48:00,670 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42245,1732085279893-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,670 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,670 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.Replication(171): 61c2066a8a0f,42245,1732085279893 started
2024-11-20T06:48:00,689 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:00,689 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,42245,1732085279893, RpcServer on 61c2066a8a0f/172.17.0.2:42245, sessionid=0x10158497e850001
2024-11-20T06:48:00,689 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:48:00,689 DEBUG [RS:0;61c2066a8a0f:42245 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,689 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,42245,1732085279893'
2024-11-20T06:48:00,689 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:48:00,690 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:48:00,690 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:48:00,690 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:48:00,691 DEBUG [RS:0;61c2066a8a0f:42245 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,691 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,42245,1732085279893'
2024-11-20T06:48:00,691 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:48:00,691 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:48:00,691 DEBUG [RS:0;61c2066a8a0f:42245 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:48:00,691 INFO  [RS:0;61c2066a8a0f:42245 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:48:00,692 INFO  [RS:0;61c2066a8a0f:42245 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:48:00,709 WARN  [61c2066a8a0f:43353 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:48:00,794 INFO  [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C42245%2C1732085279893, suffix=, logDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893, archiveDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs, maxLogs=32
2024-11-20T06:48:00,795 INFO  [RS:0;61c2066a8a0f:42245 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:00,808 INFO  [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:00,827 DEBUG [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45889:45889),(127.0.0.1/127.0.0.1:37367:37367)]
2024-11-20T06:48:00,959 DEBUG [61c2066a8a0f:43353 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:48:00,960 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:00,961 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,42245,1732085279893, state=OPENING
2024-11-20T06:48:00,992 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:48:01,003 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:01,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:01,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:01,004 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:48:01,004 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42245,1732085279893}]
2024-11-20T06:48:01,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:01,157 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:48:01,159 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:48:01,164 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:48:01,164 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:01,166 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C42245%2C1732085279893.meta, suffix=.meta, logDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893, archiveDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs, maxLogs=32
2024-11-20T06:48:01,167 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta
2024-11-20T06:48:01,176 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta
2024-11-20T06:48:01,177 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45889:45889),(127.0.0.1/127.0.0.1:37367:37367)]
2024-11-20T06:48:01,178 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:01,178 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:48:01,178 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:48:01,178 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:48:01,178 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:48:01,178 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:01,179 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:48:01,179 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:48:01,183 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:48:01,184 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:48:01,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,184 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:01,185 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:48:01,185 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:48:01,185 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:01,186 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:48:01,187 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:48:01,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,187 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:01,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:48:01,188 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:48:01,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,189 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:01,189 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:48:01,190 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740
2024-11-20T06:48:01,191 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740
2024-11-20T06:48:01,193 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:48:01,193 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:48:01,194 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:48:01,195 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:48:01,196 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734201, jitterRate=-0.06641562283039093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:48:01,196 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:48:01,197 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085281179Writing region info on filesystem at 1732085281179Initializing all the Stores at 1732085281180 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085281180Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085281182 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085281182Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085281182Cleaning up temporary data from old regions at 1732085281193 (+11 ms)Running coprocessor post-open hooks at 1732085281196 (+3 ms)Region opened successfully at 1732085281197 (+1 ms)
2024-11-20T06:48:01,199 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085281157
2024-11-20T06:48:01,201 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:48:01,201 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:48:01,202 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:01,204 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,42245,1732085279893, state=OPEN
2024-11-20T06:48:01,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:48:01,261 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:48:01,261 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:01,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:01,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:01,264 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:48:01,264 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42245,1732085279893 in 257 msec
2024-11-20T06:48:01,268 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:48:01,268 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 708 msec
2024-11-20T06:48:01,269 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:01,269 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:48:01,270 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:48:01,271 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,42245,1732085279893, seqNum=-1]
2024-11-20T06:48:01,271 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:48:01,272 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51517, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:48:01,278 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 803 msec
2024-11-20T06:48:01,278 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085281278, completionTime=-1
2024-11-20T06:48:01,279 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:48:01,279 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085341281
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085401281
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:43353, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,281 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,282 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,283 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.273sec
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:48:01,286 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:48:01,288 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:48:01,288 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:48:01,288 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43353,1732085279706-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:01,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:01,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d428c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:01,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,43353,-1 for getting cluster id
2024-11-20T06:48:01,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:48:01,371 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cb98945a-4b24-46e5-897b-4eaa0e2db404'
2024-11-20T06:48:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:48:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cb98945a-4b24-46e5-897b-4eaa0e2db404"

2024-11-20T06:48:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710e222e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,43353,-1]
2024-11-20T06:48:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:48:01,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:01,374 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:48:01,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cbeb460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:01,376 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:48:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,42245,1732085279893, seqNum=-1]
2024-11-20T06:48:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:48:01,380 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:48:01,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:01,382 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:01,385 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:48:01,385 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-20T06:48:01,385 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-20T06:48:01,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T06:48:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1807313d
2024-11-20T06:48:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T06:48:01,389 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T06:48:01,389 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T06:48:01,390 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T06:48:01,390 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:48:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T06:48:01,393 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T06:48:01,393 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,393 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-20T06:48:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:48:01,394 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T06:48:01,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741835_1011 (size=395)
2024-11-20T06:48:01,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741835_1011 (size=395)
2024-11-20T06:48:01,405 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86bd8737ae0b79e193cde2337f7998a6, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7
2024-11-20T06:48:01,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34783 is added to blk_1073741836_1012 (size=78)
2024-11-20T06:48:01,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43607 is added to blk_1073741836_1012 (size=78)
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 86bd8737ae0b79e193cde2337f7998a6, disabling compactions & flushes
2024-11-20T06:48:01,416 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6. after waiting 0 ms
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,416 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,416 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86bd8737ae0b79e193cde2337f7998a6:
Waiting for close lock at 1732085281416Disabling compacts and flushes for region at 1732085281416Disabling writes for close at 1732085281416Writing region close event to WAL at 1732085281416Closed at 1732085281416
2024-11-20T06:48:01,417 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T06:48:01,418 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732085281417"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085281417"}]},"ts":"1732085281417"}
2024-11-20T06:48:01,420 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T06:48:01,422 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T06:48:01,422 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085281422"}]},"ts":"1732085281422"}
2024-11-20T06:48:01,425 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-20T06:48:01,425 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=86bd8737ae0b79e193cde2337f7998a6, ASSIGN}]
2024-11-20T06:48:01,427 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=86bd8737ae0b79e193cde2337f7998a6, ASSIGN
2024-11-20T06:48:01,428 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=86bd8737ae0b79e193cde2337f7998a6, ASSIGN; state=OFFLINE, location=61c2066a8a0f,42245,1732085279893; forceNewPlan=false, retain=false
2024-11-20T06:48:01,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:01,579 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86bd8737ae0b79e193cde2337f7998a6, regionState=OPENING, regionLocation=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:01,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=86bd8737ae0b79e193cde2337f7998a6, ASSIGN because future has completed
2024-11-20T06:48:01,583 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86bd8737ae0b79e193cde2337f7998a6, server=61c2066a8a0f,42245,1732085279893}]
2024-11-20T06:48:01,741 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,741 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 86bd8737ae0b79e193cde2337f7998a6, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:01,741 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,742 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:01,742 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,742 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,743 INFO  [StoreOpener-86bd8737ae0b79e193cde2337f7998a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 86bd8737ae0b79e193cde2337f7998a6 
2024-11-20T06:48:01,745 INFO  [StoreOpener-86bd8737ae0b79e193cde2337f7998a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86bd8737ae0b79e193cde2337f7998a6 columnFamilyName info
2024-11-20T06:48:01,745 DEBUG [StoreOpener-86bd8737ae0b79e193cde2337f7998a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:01,746 INFO  [StoreOpener-86bd8737ae0b79e193cde2337f7998a6-1 {}] regionserver.HStore(327): Store=86bd8737ae0b79e193cde2337f7998a6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:01,746 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,747 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,747 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,748 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,748 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,750 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,752 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:01,753 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 86bd8737ae0b79e193cde2337f7998a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=877983, jitterRate=0.11641368269920349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:48:01,753 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:01,753 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 86bd8737ae0b79e193cde2337f7998a6:
Running coprocessor pre-open hook at 1732085281742Writing region info on filesystem at 1732085281742Initializing all the Stores at 1732085281743 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085281743Cleaning up temporary data from old regions at 1732085281748 (+5 ms)Running coprocessor post-open hooks at 1732085281753 (+5 ms)Region opened successfully at 1732085281753
2024-11-20T06:48:01,754 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6., pid=6, masterSystemTime=1732085281737
2024-11-20T06:48:01,757 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,757 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:01,758 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=86bd8737ae0b79e193cde2337f7998a6, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:01,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86bd8737ae0b79e193cde2337f7998a6, server=61c2066a8a0f,42245,1732085279893 because future has completed
2024-11-20T06:48:01,765 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T06:48:01,765 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 86bd8737ae0b79e193cde2337f7998a6, server=61c2066a8a0f,42245,1732085279893 in 179 msec
2024-11-20T06:48:01,768 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T06:48:01,768 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=86bd8737ae0b79e193cde2337f7998a6, ASSIGN in 340 msec
2024-11-20T06:48:01,769 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T06:48:01,769 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085281769"}]},"ts":"1732085281769"}
2024-11-20T06:48:01,771 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-20T06:48:01,772 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T06:48:01,775 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 382 msec
2024-11-20T06:48:02,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:02,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:03,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:48:03,217 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T06:48:03,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T06:48:03,218 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-20T06:48:03,219 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:48:03,219 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T06:48:03,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:03,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:04,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:04,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:05,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:05,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:06,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:06,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:06,681 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:48:06,705 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,707 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,716 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:06,721 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:48:06,722 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-20T06:48:07,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:07,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:08,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:08,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:09,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:09,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:10,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:10,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:11,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:48:11,451 INFO  [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-20T06:48:11,451 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-20T06:48:11,454 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T06:48:11,454 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6., hostname=61c2066a8a0f,42245,1732085279893, seqNum=2]
2024-11-20T06:48:11,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:12,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:12,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:13,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:13,462 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:13,463 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,463 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,464 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK], DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]) is bad.
2024-11-20T06:48:13,464 WARN  [PacketResponder: BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43607] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,464 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,465 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK], DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]) is bad.
2024-11-20T06:48:13,465 WARN  [PacketResponder: BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43607] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,465 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK], DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43607,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]) is bad.
2024-11-20T06:48:13,465 WARN  [PacketResponder: BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:43607] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:51818 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51818 dst: /127.0.0.1:34783
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2017340616_22 at /127.0.0.1:51800 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51800 dst: /127.0.0.1:34783
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:51814 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51814 dst: /127.0.0.1:34783
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:39456 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43607:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39456 dst: /127.0.0.1:43607
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:39436 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43607:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39436 dst: /127.0.0.1:43607
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2017340616_22 at /127.0.0.1:39400 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43607:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39400 dst: /127.0.0.1:43607
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:13,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77c83f30{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:13,501 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54dbaae8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:13,501 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:13,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23accf28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:13,501 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47946b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:13,503 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:13,503 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:13,503 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid 7bd97968-b92a-4691-8b80-31fce44f696d) service to localhost/127.0.0.1:41921
2024-11-20T06:48:13,503 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:13,504 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data3/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:13,504 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data4/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:13,504 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:13,512 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:13,516 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:13,517 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:13,517 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:13,517 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:48:13,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@167fd01b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:13,518 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71e80301{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:13,628 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d151a18{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-38043-hadoop-hdfs-3_4_1-tests_jar-_-any-15986669567856208319/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:13,629 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e3b369c{HTTP/1.1, (http/1.1)}{localhost:38043}
2024-11-20T06:48:13,629 INFO  [Time-limited test {}] server.Server(415): Started @176971ms
2024-11-20T06:48:13,630 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:13,667 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,667 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:34288 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34288 dst: /127.0.0.1:34783
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:34292 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34292 dst: /127.0.0.1:34783
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,667 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:13,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2017340616_22 at /127.0.0.1:34300 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34300 dst: /127.0.0.1:34783
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:13,671 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44e11af1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:13,672 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@60043bb8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:13,672 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:13,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@301612e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:13,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13646a74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:13,673 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:13,673 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:13,673 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid bbf9bc49-e85e-413d-948f-a2c21535b4d5) service to localhost/127.0.0.1:41921
2024-11-20T06:48:13,673 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:13,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data1/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:13,674 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data2/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:13,674 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:13,682 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:13,687 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:13,687 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:13,688 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:13,688 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:48:13,688 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2026736d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:13,689 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ae00177{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:13,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2392cae{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-46139-hadoop-hdfs-3_4_1-tests_jar-_-any-8475300592826139741/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:13,819 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29d5ccf{HTTP/1.1, (http/1.1)}{localhost:46139}
2024-11-20T06:48:13,819 INFO  [Time-limited test {}] server.Server(415): Started @177161ms
2024-11-20T06:48:13,820 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:14,093 WARN  [Thread-1348 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:14,096 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c8a0e661597d80c with lease ID 0x381b7ee14fd19909: from storage DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8 node DatanodeRegistration(127.0.0.1:39361, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=40443, infoSecurePort=0, ipcPort=44961, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:48:14,096 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3c8a0e661597d80c with lease ID 0x381b7ee14fd19909: from storage DS-94c52b76-c37a-4a58-93de-fa39d0ee8486 node DatanodeRegistration(127.0.0.1:39361, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=40443, infoSecurePort=0, ipcPort=44961, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:14,258 WARN  [Thread-1368 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:14,261 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6441b7e6446c477 with lease ID 0x381b7ee14fd1990a: from storage DS-e2ff7347-8779-4a65-b08d-5c101395526b node DatanodeRegistration(127.0.0.1:34433, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=39103, infoSecurePort=0, ipcPort=45539, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:14,261 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6441b7e6446c477 with lease ID 0x381b7ee14fd1990a: from storage DS-aeb9a505-6d1e-4abf-ace3-c8b3a351ed52 node DatanodeRegistration(127.0.0.1:34433, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=39103, infoSecurePort=0, ipcPort=45539, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:14,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:14,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:14,845 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-20T06:48:14,848 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-20T06:48:14,850 ERROR [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:14,850 WARN  [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:14,850 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C42245%2C1732085279893:(num 1732085280795) roll requested
2024-11-20T06:48:14,850 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:14,857 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 newFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:14,857 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:14,857 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:14,857 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:14,857 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:14,857 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:14,858 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:14,858 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:14,858 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:14,858 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:14,858 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40443:40443),(127.0.0.1/127.0.0.1:39103:39103)]
2024-11-20T06:48:14,858 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 is not closed yet, will try archiving it next time
2024-11-20T06:48:14,859 WARN  [IPC Server handler 4 on default port 41921 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-11-20T06:48:14,859 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 after 1ms
2024-11-20T06:48:15,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:15,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:16,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:16,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:16,863 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-20T06:48:17,095 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T06:48:17,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:17,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:18,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:18,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:18,860 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 after 4002ms
2024-11-20T06:48:18,868 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:34433,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:18,869 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39361,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK], DatanodeInfoWithStorage[127.0.0.1:34433,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34433,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]) is bad.
2024-11-20T06:48:18,869 WARN  [PacketResponder: BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:34433] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:18,869 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:54472 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54472 dst: /127.0.0.1:39361
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:18,870 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:60196 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:34433:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60196 dst: /127.0.0.1:34433
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:18,913 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2392cae{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:18,914 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29d5ccf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:18,914 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:18,914 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ae00177{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:18,914 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2026736d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:18,915 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:18,915 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:18,915 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:18,915 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid bbf9bc49-e85e-413d-948f-a2c21535b4d5) service to localhost/127.0.0.1:41921
2024-11-20T06:48:18,916 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data1/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:18,916 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data2/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:18,916 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:18,923 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:18,927 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:18,928 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:18,928 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:18,928 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:48:18,928 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7269a538{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:18,929 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7790ff99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:19,031 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b7e8c9e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-45223-hadoop-hdfs-3_4_1-tests_jar-_-any-964020986996486620/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:19,031 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58c81822{HTTP/1.1, (http/1.1)}{localhost:45223}
2024-11-20T06:48:19,031 INFO  [Time-limited test {}] server.Server(415): Started @182373ms
2024-11-20T06:48:19,032 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:19,079 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:19,080 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1038324930_22 at /127.0.0.1:54484 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39361:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54484 dst: /127.0.0.1:39361
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:19,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d151a18{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:19,084 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e3b369c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:19,084 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:19,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71e80301{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:19,084 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@167fd01b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:19,085 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:19,085 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:19,085 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:19,085 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid 7bd97968-b92a-4691-8b80-31fce44f696d) service to localhost/127.0.0.1:41921
2024-11-20T06:48:19,086 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data3/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:19,086 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data4/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:19,086 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:19,103 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:19,106 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:19,107 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:19,107 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:19,107 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:48:19,107 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ddd02f5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:19,108 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10215f32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:19,211 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@121dd606{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/java.io.tmpdir/jetty-localhost-34773-hadoop-hdfs-3_4_1-tests_jar-_-any-254170975874407388/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:19,211 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7202b0aa{HTTP/1.1, (http/1.1)}{localhost:34773}
2024-11-20T06:48:19,211 INFO  [Time-limited test {}] server.Server(415): Started @182553ms
2024-11-20T06:48:19,212 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:19,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:19,462 WARN  [Thread-1422 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:19,464 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80203e918e2a9272 with lease ID 0x381b7ee14fd1990b: from storage DS-e2ff7347-8779-4a65-b08d-5c101395526b node DatanodeRegistration(127.0.0.1:44675, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=42727, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:19,464 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80203e918e2a9272 with lease ID 0x381b7ee14fd1990b: from storage DS-aeb9a505-6d1e-4abf-ace3-c8b3a351ed52 node DatanodeRegistration(127.0.0.1:44675, datanodeUuid=bbf9bc49-e85e-413d-948f-a2c21535b4d5, infoPort=42727, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:48:19,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:19,594 WARN  [Thread-1442 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:19,597 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25764b082240ef23 with lease ID 0x381b7ee14fd1990c: from storage DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8 node DatanodeRegistration(127.0.0.1:42739, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=41363, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:19,597 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25764b082240ef23 with lease ID 0x381b7ee14fd1990c: from storage DS-94c52b76-c37a-4a58-93de-fa39d0ee8486 node DatanodeRegistration(127.0.0.1:42739, datanodeUuid=7bd97968-b92a-4691-8b80-31fce44f696d, infoPort=41363, infoSecurePort=0, ipcPort=41111, storageInfo=lv=-57;cid=testClusterID;nsid=2011601740;c=1732085277870), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:20,248 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-20T06:48:20,251 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-20T06:48:20,253 ERROR [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39361,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:20,253 WARN  [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39361,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:20,254 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C42245%2C1732085279893:(num 1732085294850) roll requested
2024-11-20T06:48:20,254 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:20,263 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 newFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:20,264 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:20,264 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:20,264 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:20,264 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:20,264 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:20,264 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:20,264 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39361,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:20,265 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39361,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:20,265 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:20,265 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42727:42727),(127.0.0.1/127.0.0.1:41363:41363)]
2024-11-20T06:48:20,265 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 is not closed yet, will try archiving it next time
2024-11-20T06:48:20,265 WARN  [IPC Server handler 3 on default port 41921 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-20T06:48:20,266 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 after 1ms
2024-11-20T06:48:20,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:20,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:21,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:21,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:22,267 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:22,272 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254 newFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:22,272 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:22,272 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:22,272 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:22,272 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:22,272 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:22,273 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:22,274 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741838_1019 (size=1264)
2024-11-20T06:48:22,274 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741838_1019 (size=1264)
2024-11-20T06:48:22,275 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 is not closed yet, will try archiving it next time
2024-11-20T06:48:22,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:42727:42727)]
2024-11-20T06:48:22,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 is not closed yet, will try archiving it next time
2024-11-20T06:48:22,283 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:22,283 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:22,283 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795 after 0ms
2024-11-20T06:48:22,283 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:22,294 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732085281753/Put/vlen=218/seqid=0]
2024-11-20T06:48:22,294 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732085291460/Put/vlen=1045/seqid=0]
2024-11-20T06:48:22,294 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085280795
2024-11-20T06:48:22,294 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:22,294 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:22,295 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 after 1ms
2024-11-20T06:48:22,295 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:22,299 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732085294849/Put/vlen=1045/seqid=0]
2024-11-20T06:48:22,299 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732085296865/Put/vlen=1045/seqid=0]
2024-11-20T06:48:22,299 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850
2024-11-20T06:48:22,299 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:22,299 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:22,300 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254 after 1ms
2024-11-20T06:48:22,300 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085300254
2024-11-20T06:48:22,304 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732085300253/Put/vlen=1045/seqid=0]
2024-11-20T06:48:22,304 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:22,304 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:22,304 WARN  [IPC Server handler 0 on default port 41921 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-20T06:48:22,305 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 after 1ms
2024-11-20T06:48:22,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:22,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:22,599 WARN  [ResponseProcessor for block BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:22,599 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2017340616_22 at /127.0.0.1:41998 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42739:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41998 dst: /127.0.0.1:42739
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42739 remote=/127.0.0.1:41998]. Total timeout mills is 60000, 59672 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:22,600 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2017340616_22 at /127.0.0.1:60420 [Receiving block BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44675:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60420 dst: /127.0.0.1:44675
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:22,600 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 block BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42739,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK], DatanodeInfoWithStorage[127.0.0.1:44675,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42739,DS-a8c91f36-1165-4066-9a8a-7a49d768aaa8,DISK]) is bad.
2024-11-20T06:48:22,600 WARN  [DataStreamer for file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 block BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:22,607 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741839_1022 (size=85)
2024-11-20T06:48:22,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741839_1022 (size=85)
2024-11-20T06:48:23,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:23,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:24,267 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085294850 after 4002ms
2024-11-20T06:48:24,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:24,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:25,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:25,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:26,305 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 after 4001ms
2024-11-20T06:48:26,305 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:26,309 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:26,310 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 86bd8737ae0b79e193cde2337f7998a6 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-20T06:48:26,310 ERROR [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,311 WARN  [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,311 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C42245%2C1732085279893:(num 1732085302266) roll requested
2024-11-20T06:48:26,311 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.1732085306311
2024-11-20T06:48:26,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:26,324 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 newFile=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085306311
2024-11-20T06:48:26,324 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,324 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,325 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,325 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,325 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,325 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085306311
2024-11-20T06:48:26,325 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,326 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:42727:42727)]
2024-11-20T06:48:26,326 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 is not closed yet, will try archiving it next time
2024-11-20T06:48:26,325 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1488376784-172.17.0.2-1732085277870:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,326 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:26,326 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 after 0ms
2024-11-20T06:48:26,327 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.1732085302266 to hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs/61c2066a8a0f%2C42245%2C1732085279893.1732085302266
2024-11-20T06:48:26,348 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/.tmp/info/e651f00961fc4c35951f4bb0336bc136 is 1080, key is row1002/info:/1732085291460/Put/seqid=0
2024-11-20T06:48:26,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741841_1024 (size=9270)
2024-11-20T06:48:26,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741841_1024 (size=9270)
2024-11-20T06:48:26,464 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T06:48:26,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:26,770 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/.tmp/info/e651f00961fc4c35951f4bb0336bc136
2024-11-20T06:48:26,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/.tmp/info/e651f00961fc4c35951f4bb0336bc136 as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/info/e651f00961fc4c35951f4bb0336bc136
2024-11-20T06:48:26,786 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/info/e651f00961fc4c35951f4bb0336bc136, entries=4, sequenceid=8, filesize=9.1 K
2024-11-20T06:48:26,788 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 86bd8737ae0b79e193cde2337f7998a6 in 477ms, sequenceid=8, compaction requested=false
2024-11-20T06:48:26,788 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 86bd8737ae0b79e193cde2337f7998a6:

2024-11-20T06:48:26,788 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-20T06:48:26,788 ERROR [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,788 WARN  [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7-prefix:61c2066a8a0f,42245,1732085279893.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,788 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C42245%2C1732085279893.meta:.meta(num 1732085281167) roll requested
2024-11-20T06:48:26,789 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42245%2C1732085279893.meta.1732085306788.meta
2024-11-20T06:48:26,794 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,794 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,794 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,794 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,794 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:26,794 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085306788.meta
2024-11-20T06:48:26,795 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,795 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:26,795 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta
2024-11-20T06:48:26,795 WARN  [IPC Server handler 2 on default port 41921 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-11-20T06:48:26,796 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta after 1ms
2024-11-20T06:48:26,799 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42727:42727),(127.0.0.1/127.0.0.1:41363:41363)]
2024-11-20T06:48:26,799 DEBUG [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta is not closed yet, will try archiving it next time
2024-11-20T06:48:26,817 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/info/a4a61b0f050140bbb1dc7808018db03b is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6./info:regioninfo/1732085281758/Put/seqid=0
2024-11-20T06:48:26,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741843_1027 (size=7125)
2024-11-20T06:48:26,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741843_1027 (size=7125)
2024-11-20T06:48:26,829 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/info/a4a61b0f050140bbb1dc7808018db03b
2024-11-20T06:48:26,849 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/ns/6cc8bbbec2ce4d30be70957618da86ff is 43, key is default/ns:d/1732085281273/Put/seqid=0
2024-11-20T06:48:26,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741844_1028 (size=5153)
2024-11-20T06:48:26,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741844_1028 (size=5153)
2024-11-20T06:48:26,855 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/ns/6cc8bbbec2ce4d30be70957618da86ff
2024-11-20T06:48:26,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/table/09008445fe794b269f16c92274e5dbda is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732085281769/Put/seqid=0
2024-11-20T06:48:26,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741845_1029 (size=5438)
2024-11-20T06:48:26,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741845_1029 (size=5438)
2024-11-20T06:48:26,881 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/table/09008445fe794b269f16c92274e5dbda
2024-11-20T06:48:26,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/info/a4a61b0f050140bbb1dc7808018db03b as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/info/a4a61b0f050140bbb1dc7808018db03b
2024-11-20T06:48:26,894 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/info/a4a61b0f050140bbb1dc7808018db03b, entries=10, sequenceid=11, filesize=7.0 K
2024-11-20T06:48:26,895 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/ns/6cc8bbbec2ce4d30be70957618da86ff as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/ns/6cc8bbbec2ce4d30be70957618da86ff
2024-11-20T06:48:26,901 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/ns/6cc8bbbec2ce4d30be70957618da86ff, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T06:48:26,902 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/.tmp/table/09008445fe794b269f16c92274e5dbda as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/table/09008445fe794b269f16c92274e5dbda
2024-11-20T06:48:26,909 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/table/09008445fe794b269f16c92274e5dbda, entries=2, sequenceid=11, filesize=5.3 K
2024-11-20T06:48:26,911 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=11, compaction requested=false
2024-11-20T06:48:26,911 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-20T06:48:26,917 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:48:26,917 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:48:26,917 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:48:26,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:26,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:26,918 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:48:26,918 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:48:26,918 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=182522426, stopped=false
2024-11-20T06:48:26,918 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,43353,1732085279706
2024-11-20T06:48:26,953 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:26,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:26,953 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:26,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:26,953 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:48:26,954 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:48:26,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:48:26,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:48:26,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:26,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:48:26,954 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,42245,1732085279893' *****
2024-11-20T06:48:26,954 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:48:26,954 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:48:26,954 INFO  [RS:0;61c2066a8a0f:42245 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:48:26,954 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:48:26,954 INFO  [RS:0;61c2066a8a0f:42245 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(3091): Received CLOSE for 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:42245.
2024-11-20T06:48:26,955 DEBUG [RS:0;61c2066a8a0f:42245 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:48:26,955 DEBUG [RS:0;61c2066a8a0f:42245 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:26,955 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 86bd8737ae0b79e193cde2337f7998a6, disabling compactions & flushes
2024-11-20T06:48:26,955 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:26,955 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:48:26,955 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6. after waiting 0 ms
2024-11-20T06:48:26,955 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:48:26,955 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:26,956 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T06:48:26,956 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1325): Online Regions={86bd8737ae0b79e193cde2337f7998a6=TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:48:26,956 DEBUG [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 86bd8737ae0b79e193cde2337f7998a6
2024-11-20T06:48:26,956 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:48:26,956 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:48:26,956 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:48:26,956 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:48:26,956 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:48:26,960 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/default/TestLogRolling-testLogRollOnPipelineRestart/86bd8737ae0b79e193cde2337f7998a6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-20T06:48:26,961 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:26,961 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 86bd8737ae0b79e193cde2337f7998a6:
Waiting for close lock at 1732085306955Running coprocessor pre-close hooks at 1732085306955Disabling compacts and flushes for region at 1732085306955Disabling writes for close at 1732085306955Writing region close event to WAL at 1732085306956 (+1 ms)Running coprocessor post-close hooks at 1732085306960 (+4 ms)Closed at 1732085306960
2024-11-20T06:48:26,961 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732085281389.86bd8737ae0b79e193cde2337f7998a6.
2024-11-20T06:48:26,961 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T06:48:26,962 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:48:26,962 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:48:26,962 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085306956Running coprocessor pre-close hooks at 1732085306956Disabling compacts and flushes for region at 1732085306956Disabling writes for close at 1732085306956Writing region close event to WAL at 1732085306957 (+1 ms)Running coprocessor post-close hooks at 1732085306962 (+5 ms)Closed at 1732085306962
2024-11-20T06:48:26,962 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:48:27,156 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,42245,1732085279893; all regions closed.
2024-11-20T06:48:27,156 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:27,156 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:27,157 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:27,157 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:27,157 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:27,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741842_1025 (size=825)
2024-11-20T06:48:27,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741842_1025 (size=825)
2024-11-20T06:48:27,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:27,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:27,651 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-20T06:48:27,651 INFO  [regionserver/61c2066a8a0f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-20T06:48:28,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:28,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:28,598 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T06:48:28,656 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:48:29,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:29,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:29,678 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:48:30,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:30,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:30,796 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta after 4001ms
2024-11-20T06:48:30,797 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/WALs/61c2066a8a0f,42245,1732085279893/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta to hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs/61c2066a8a0f%2C42245%2C1732085279893.meta.1732085281167.meta
2024-11-20T06:48:30,801 DEBUG [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs
2024-11-20T06:48:30,802 INFO  [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C42245%2C1732085279893.meta:.meta(num 1732085306788)
2024-11-20T06:48:30,802 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,802 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,803 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,803 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,803 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741840_1023 (size=1162)
2024-11-20T06:48:30,806 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741840_1023 (size=1162)
2024-11-20T06:48:30,813 DEBUG [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C42245%2C1732085279893:(num 1732085306311)
2024-11-20T06:48:30,813 DEBUG [RS:0;61c2066a8a0f:42245 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:48:30,813 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:48:30,813 INFO  [RS:0;61c2066a8a0f:42245 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42245
2024-11-20T06:48:30,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:48:30,861 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,42245,1732085279893
2024-11-20T06:48:30,861 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:48:30,861 ERROR [pool-525-thread-1-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher.
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$362/0x00007fabc49009d8@1dd595da rejected from java.util.concurrent.ThreadPoolExecutor@58d70804[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?]
	at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4]
2024-11-20T06:48:30,869 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,42245,1732085279893]
2024-11-20T06:48:30,877 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,42245,1732085279893 already deleted, retry=false
2024-11-20T06:48:30,877 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,42245,1732085279893 expired; onlineServers=0
2024-11-20T06:48:30,877 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,43353,1732085279706' *****
2024-11-20T06:48:30,877 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:48:30,877 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:48:30,877 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:48:30,878 DEBUG [M:0;61c2066a8a0f:43353 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:48:30,878 DEBUG [M:0;61c2066a8a0f:43353 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:48:30,878 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:48:30,878 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085280484 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085280484,5,FailOnTimeoutGroup]
2024-11-20T06:48:30,878 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085280484 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085280484,5,FailOnTimeoutGroup]
2024-11-20T06:48:30,878 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:48:30,878 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:48:30,878 DEBUG [M:0;61c2066a8a0f:43353 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:48:30,878 INFO  [M:0;61c2066a8a0f:43353 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:48:30,878 INFO  [M:0;61c2066a8a0f:43353 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:48:30,879 INFO  [M:0;61c2066a8a0f:43353 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:48:30,879 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:48:30,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:48:30,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:30,886 DEBUG [M:0;61c2066a8a0f:43353 {}] zookeeper.ZKUtil(347): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:48:30,886 WARN  [M:0;61c2066a8a0f:43353 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:48:30,887 INFO  [M:0;61c2066a8a0f:43353 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/.lastflushedseqids
2024-11-20T06:48:30,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741846_1030 (size=111)
2024-11-20T06:48:30,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741846_1030 (size=111)
2024-11-20T06:48:30,899 INFO  [M:0;61c2066a8a0f:43353 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:48:30,899 INFO  [M:0;61c2066a8a0f:43353 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:48:30,899 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:48:30,899 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:30,899 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:30,899 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:48:30,899 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:30,899 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-20T06:48:30,900 ERROR [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData-prefix:61c2066a8a0f,43353,1732085279706 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:30,900 WARN  [FSHLog-0-hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData-prefix:61c2066a8a0f,43353,1732085279706 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:30,900 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 61c2066a8a0f%2C43353%2C1732085279706:(num 1732085280174) roll requested
2024-11-20T06:48:30,900 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C43353%2C1732085279706.1732085310900
2024-11-20T06:48:30,905 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,906 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,906 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,906 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,906 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:30,906 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085310900
2024-11-20T06:48:30,906 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:30,907 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34783,DS-e2ff7347-8779-4a65-b08d-5c101395526b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-20T06:48:30,907 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174
2024-11-20T06:48:30,907 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42727:42727),(127.0.0.1/127.0.0.1:41363:41363)]
2024-11-20T06:48:30,907 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 is not closed yet, will try archiving it next time
2024-11-20T06:48:30,907 WARN  [IPC Server handler 1 on default port 41921 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-20T06:48:30,908 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 after 1ms
2024-11-20T06:48:30,923 DEBUG [M:0;61c2066a8a0f:43353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d761cd1e14a24a34bf3ea23ea283ce16 is 82, key is hbase:meta,,1/info:regioninfo/1732085281202/Put/seqid=0
2024-11-20T06:48:30,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741848_1033 (size=5672)
2024-11-20T06:48:30,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741848_1033 (size=5672)
2024-11-20T06:48:30,928 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d761cd1e14a24a34bf3ea23ea283ce16
2024-11-20T06:48:30,948 DEBUG [M:0;61c2066a8a0f:43353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43890429f6b54aab9c90752b80f288ae is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085281774/Put/seqid=0
2024-11-20T06:48:30,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741849_1034 (size=6118)
2024-11-20T06:48:30,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741849_1034 (size=6118)
2024-11-20T06:48:30,952 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43890429f6b54aab9c90752b80f288ae
2024-11-20T06:48:30,969 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:48:30,969 INFO  [RS:0;61c2066a8a0f:42245 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:48:30,969 DEBUG [pool-525-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42245-0x10158497e850001, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:48:30,969 INFO  [RS:0;61c2066a8a0f:42245 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,42245,1732085279893; zookeeper connection closed.
2024-11-20T06:48:30,969 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77eee0de {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77eee0de
2024-11-20T06:48:30,970 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:48:30,978 DEBUG [M:0;61c2066a8a0f:43353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2948ee143baf4a04a4bda542c22d0938 is 69, key is 61c2066a8a0f,42245,1732085279893/rs:state/1732085280615/Put/seqid=0
2024-11-20T06:48:30,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741850_1035 (size=5156)
2024-11-20T06:48:30,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741850_1035 (size=5156)
2024-11-20T06:48:30,983 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2948ee143baf4a04a4bda542c22d0938
2024-11-20T06:48:31,003 DEBUG [M:0;61c2066a8a0f:43353 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57372265b8a1432ab9e05b9374ead114 is 52, key is load_balancer_on/state:d/1732085281384/Put/seqid=0
2024-11-20T06:48:31,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741851_1036 (size=5056)
2024-11-20T06:48:31,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741851_1036 (size=5056)
2024-11-20T06:48:31,008 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57372265b8a1432ab9e05b9374ead114
2024-11-20T06:48:31,014 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d761cd1e14a24a34bf3ea23ea283ce16 as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d761cd1e14a24a34bf3ea23ea283ce16
2024-11-20T06:48:31,020 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d761cd1e14a24a34bf3ea23ea283ce16, entries=8, sequenceid=56, filesize=5.5 K
2024-11-20T06:48:31,021 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43890429f6b54aab9c90752b80f288ae as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/43890429f6b54aab9c90752b80f288ae
2024-11-20T06:48:31,027 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/43890429f6b54aab9c90752b80f288ae, entries=6, sequenceid=56, filesize=6.0 K
2024-11-20T06:48:31,028 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2948ee143baf4a04a4bda542c22d0938 as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2948ee143baf4a04a4bda542c22d0938
2024-11-20T06:48:31,033 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2948ee143baf4a04a4bda542c22d0938, entries=1, sequenceid=56, filesize=5.0 K
2024-11-20T06:48:31,034 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57372265b8a1432ab9e05b9374ead114 as hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/57372265b8a1432ab9e05b9374ead114
2024-11-20T06:48:31,040 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/57372265b8a1432ab9e05b9374ead114, entries=1, sequenceid=56, filesize=4.9 K
2024-11-20T06:48:31,041 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=56, compaction requested=false
2024-11-20T06:48:31,043 INFO  [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:31,043 DEBUG [M:0;61c2066a8a0f:43353 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085310899Disabling compacts and flushes for region at 1732085310899Disabling writes for close at 1732085310899Obtaining lock to block concurrent updates at 1732085310899Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085310899Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732085310900 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085310908 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085310908Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085310922 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085310922Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085310933 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085310947 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085310947Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085310957 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085310977 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085310977Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085310988 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085311002 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085311002Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3daf3635: reopening flushed file at 1732085311013 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@95f49e4: reopening flushed file at 1732085311021 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@538dcee9: reopening flushed file at 1732085311027 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a844eed: reopening flushed file at 1732085311033 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=56, compaction requested=false at 1732085311041 (+8 ms)Writing region close event to WAL at 1732085311043 (+2 ms)Closed at 1732085311043
2024-11-20T06:48:31,043 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:31,043 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:31,043 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:31,043 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:31,043 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:48:31,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42739 is added to blk_1073741847_1031 (size=757)
2024-11-20T06:48:31,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44675 is added to blk_1073741847_1031 (size=757)
2024-11-20T06:48:31,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:31,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:31,597 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-20T06:48:31,961 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,962 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,999 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:31,999 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:32,502 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:48:32,503 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,504 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,505 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,505 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:32,533 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,533 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,533 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,534 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,534 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,534 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,539 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,539 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,539 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:32,543 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:33,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:48:33,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:48:33,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T06:48:33,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-20T06:48:33,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:33,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:34,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:34,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:34,908 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 after 4001ms
2024-11-20T06:48:34,909 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/WALs/61c2066a8a0f,43353,1732085279706/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 to hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/oldWALs/61c2066a8a0f%2C43353%2C1732085279706.1732085280174
2024-11-20T06:48:34,912 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/MasterData/oldWALs/61c2066a8a0f%2C43353%2C1732085279706.1732085280174 to hdfs://localhost:41921/user/jenkins/test-data/9ecb2861-e33b-7af4-5833-576e7c1098a7/oldWALs/61c2066a8a0f%2C43353%2C1732085279706.1732085280174$masterlocalwal$
2024-11-20T06:48:34,912 INFO  [M:0;61c2066a8a0f:43353 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:48:34,912 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:48:34,912 INFO  [M:0;61c2066a8a0f:43353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43353
2024-11-20T06:48:34,912 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:48:35,052 INFO  [M:0;61c2066a8a0f:43353 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:48:35,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:48:35,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43353-0x10158497e850000, quorum=127.0.0.1:58486, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:48:35,055 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@121dd606{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:35,055 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7202b0aa{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:35,055 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:35,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10215f32{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:35,056 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ddd02f5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:35,057 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:35,057 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:35,057 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid 7bd97968-b92a-4691-8b80-31fce44f696d) service to localhost/127.0.0.1:41921
2024-11-20T06:48:35,057 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:35,058 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data3/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:35,059 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data4/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:35,059 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:35,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b7e8c9e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:35,062 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58c81822{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:35,062 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:35,062 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7790ff99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:35,062 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7269a538{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:35,063 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:48:35,063 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:48:35,063 WARN  [BP-1488376784-172.17.0.2-1732085277870 heartbeating to localhost/127.0.0.1:41921 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1488376784-172.17.0.2-1732085277870 (Datanode Uuid bbf9bc49-e85e-413d-948f-a2c21535b4d5) service to localhost/127.0.0.1:41921
2024-11-20T06:48:35,063 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:48:35,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data1/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:35,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/cluster_78d22a3d-0ff1-3f57-7fb4-1a1dbae1d35e/data/data2/current/BP-1488376784-172.17.0.2-1732085277870 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:48:35,064 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:48:35,070 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@232fa1ae{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:48:35,071 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c07fc0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:48:35,071 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:48:35,071 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10c583a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:48:35,071 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7096145a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir/,STOPPED}
2024-11-20T06:48:35,078 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:48:35,098 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:48:35,106 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=181 (was 155)
Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:41921 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:41921 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41921
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:41921
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41921
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:41921 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 431) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=249 (was 250), ProcessCount=11 (was 11), AvailableMemoryMB=3769 (was 4034)
2024-11-20T06:48:35,113 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=181, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=249, ProcessCount=11, AvailableMemoryMB=3769
2024-11-20T06:48:35,113 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.log.dir so I do NOT create it in target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f73a0a54-e86e-c05f-933f-588e06621604/hadoop.tmp.dir so I do NOT create it in target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090, deleteOnExit=true
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/test.cache.data in system properties and HBase conf
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:48:35,114 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:48:35,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:48:35,115 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:48:35,116 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:48:35,132 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:48:35,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:35,391 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:35,396 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:35,397 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:35,397 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:35,397 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:48:35,397 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:35,398 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20e4ef1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:35,398 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fb33a9d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:35,500 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76b2d62a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/java.io.tmpdir/jetty-localhost-34849-hadoop-hdfs-3_4_1-tests_jar-_-any-12232588944773030729/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:48:35,501 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47d3f616{HTTP/1.1, (http/1.1)}{localhost:34849}
2024-11-20T06:48:35,501 INFO  [Time-limited test {}] server.Server(415): Started @198843ms
2024-11-20T06:48:35,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:35,513 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:48:35,711 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:35,714 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:35,715 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:35,715 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:35,715 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:48:35,716 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b079ea2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:35,716 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@507832d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:35,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c1ed8d4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/java.io.tmpdir/jetty-localhost-44713-hadoop-hdfs-3_4_1-tests_jar-_-any-6882832431042347430/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:35,818 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d36967f{HTTP/1.1, (http/1.1)}{localhost:44713}
2024-11-20T06:48:35,818 INFO  [Time-limited test {}] server.Server(415): Started @199160ms
2024-11-20T06:48:35,819 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:35,845 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:48:35,848 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:48:35,849 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:48:35,849 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:48:35,849 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:48:35,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b7fc8f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:48:35,850 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@311facd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:48:35,952 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@75e789f0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/java.io.tmpdir/jetty-localhost-37647-hadoop-hdfs-3_4_1-tests_jar-_-any-3381608334464441994/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:48:35,953 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@523d16c3{HTTP/1.1, (http/1.1)}{localhost:37647}
2024-11-20T06:48:35,953 INFO  [Time-limited test {}] server.Server(415): Started @199295ms
2024-11-20T06:48:35,954 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:48:36,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:36,463 WARN  [Thread-1663 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data2/current/BP-1170907024-172.17.0.2-1732085315144/current, will proceed with Du for space computation calculation, 
2024-11-20T06:48:36,463 WARN  [Thread-1662 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data1/current/BP-1170907024-172.17.0.2-1732085315144/current, will proceed with Du for space computation calculation, 
2024-11-20T06:48:36,492 WARN  [Thread-1626 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:36,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8902f2934c13f1a8 with lease ID 0x3c7ef60990a5db4e: Processing first storage report for DS-626a2e44-9b2a-4c1d-b35c-d3d6bbcbcf87 from datanode DatanodeRegistration(127.0.0.1:41231, datanodeUuid=47cfc7e8-104a-4993-8c7d-95b58c31fd3c, infoPort=34847, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144)
2024-11-20T06:48:36,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8902f2934c13f1a8 with lease ID 0x3c7ef60990a5db4e: from storage DS-626a2e44-9b2a-4c1d-b35c-d3d6bbcbcf87 node DatanodeRegistration(127.0.0.1:41231, datanodeUuid=47cfc7e8-104a-4993-8c7d-95b58c31fd3c, infoPort=34847, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:36,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8902f2934c13f1a8 with lease ID 0x3c7ef60990a5db4e: Processing first storage report for DS-b29a8dd7-ced1-4f3d-863d-46e0ea55f43a from datanode DatanodeRegistration(127.0.0.1:41231, datanodeUuid=47cfc7e8-104a-4993-8c7d-95b58c31fd3c, infoPort=34847, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144)
2024-11-20T06:48:36,494 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8902f2934c13f1a8 with lease ID 0x3c7ef60990a5db4e: from storage DS-b29a8dd7-ced1-4f3d-863d-46e0ea55f43a node DatanodeRegistration(127.0.0.1:41231, datanodeUuid=47cfc7e8-104a-4993-8c7d-95b58c31fd3c, infoPort=34847, infoSecurePort=0, ipcPort=43019, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:48:36,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:36,588 WARN  [Thread-1673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data3/current/BP-1170907024-172.17.0.2-1732085315144/current, will proceed with Du for space computation calculation, 
2024-11-20T06:48:36,588 WARN  [Thread-1674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data4/current/BP-1170907024-172.17.0.2-1732085315144/current, will proceed with Du for space computation calculation, 
2024-11-20T06:48:36,607 WARN  [Thread-1649 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:48:36,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d472b58c414343b with lease ID 0x3c7ef60990a5db4f: Processing first storage report for DS-869b0c04-cbce-4ea2-9f1f-5839164c7ec3 from datanode DatanodeRegistration(127.0.0.1:39143, datanodeUuid=05b5a924-6c82-4797-ab45-d67ddabb0bf1, infoPort=46297, infoSecurePort=0, ipcPort=42111, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144)
2024-11-20T06:48:36,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d472b58c414343b with lease ID 0x3c7ef60990a5db4f: from storage DS-869b0c04-cbce-4ea2-9f1f-5839164c7ec3 node DatanodeRegistration(127.0.0.1:39143, datanodeUuid=05b5a924-6c82-4797-ab45-d67ddabb0bf1, infoPort=46297, infoSecurePort=0, ipcPort=42111, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:36,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7d472b58c414343b with lease ID 0x3c7ef60990a5db4f: Processing first storage report for DS-3da00f77-8a2f-4b45-8534-6d4f55f7fd0b from datanode DatanodeRegistration(127.0.0.1:39143, datanodeUuid=05b5a924-6c82-4797-ab45-d67ddabb0bf1, infoPort=46297, infoSecurePort=0, ipcPort=42111, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144)
2024-11-20T06:48:36,609 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7d472b58c414343b with lease ID 0x3c7ef60990a5db4f: from storage DS-3da00f77-8a2f-4b45-8534-6d4f55f7fd0b node DatanodeRegistration(127.0.0.1:39143, datanodeUuid=05b5a924-6c82-4797-ab45-d67ddabb0bf1, infoPort=46297, infoSecurePort=0, ipcPort=42111, storageInfo=lv=-57;cid=testClusterID;nsid=1343319154;c=1732085315144), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:48:36,681 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598
2024-11-20T06:48:36,684 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/zookeeper_0, clientPort=62815, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:48:36,685 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62815
2024-11-20T06:48:36,685 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,687 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:48:36,696 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:48:36,697 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c with version=8
2024-11-20T06:48:36,697 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:48:36,699 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:48:36,700 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:48:36,701 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41615
2024-11-20T06:48:36,702 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41615 connecting to ZooKeeper ensemble=127.0.0.1:62815
2024-11-20T06:48:36,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416150x0, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:48:36,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41615-0x101584a0f100000 connected
2024-11-20T06:48:36,818 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,820 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,823 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:48:36,823 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c, hbase.cluster.distributed=false
2024-11-20T06:48:36,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:48:36,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41615
2024-11-20T06:48:36,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41615
2024-11-20T06:48:36,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41615
2024-11-20T06:48:36,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41615
2024-11-20T06:48:36,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41615
2024-11-20T06:48:36,845 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:48:36,845 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:48:36,846 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41431
2024-11-20T06:48:36,848 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41431 connecting to ZooKeeper ensemble=127.0.0.1:62815
2024-11-20T06:48:36,848 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,850 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:36,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414310x0, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:48:36,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41431-0x101584a0f100001 connected
2024-11-20T06:48:36,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:48:36,973 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:48:36,974 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:48:36,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:48:36,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:48:36,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41431
2024-11-20T06:48:36,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41431
2024-11-20T06:48:36,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41431
2024-11-20T06:48:36,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41431
2024-11-20T06:48:36,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41431
2024-11-20T06:48:36,993 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:41615
2024-11-20T06:48:36,997 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:37,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:37,007 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:48:37,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,018 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:48:37,019 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,41615,1732085316699 from backup master directory
2024-11-20T06:48:37,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:37,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:48:37,026 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:48:37,026 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,030 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/hbase.id] with ID: 8ee19e05-3c5a-43c3-9988-647353340197
2024-11-20T06:48:37,030 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/.tmp/hbase.id
2024-11-20T06:48:37,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:48:37,037 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:48:37,037 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/.tmp/hbase.id]:[hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/hbase.id]
2024-11-20T06:48:37,049 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:37,049 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:48:37,051 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-20T06:48:37,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:48:37,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:48:37,070 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:48:37,071 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:48:37,071 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:37,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:48:37,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:48:37,079 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store
2024-11-20T06:48:37,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:48:37,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:48:37,097 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:37,097 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:48:37,097 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085317097Disabling compacts and flushes for region at 1732085317097Disabling writes for close at 1732085317097Writing region close event to WAL at 1732085317097Closed at 1732085317097
2024-11-20T06:48:37,098 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/.initializing
2024-11-20T06:48:37,098 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/WALs/61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,100 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41615%2C1732085316699, suffix=, logDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/WALs/61c2066a8a0f,41615,1732085316699, archiveDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/oldWALs, maxLogs=10
2024-11-20T06:48:37,101 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41615%2C1732085316699.1732085317101
2024-11-20T06:48:37,106 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/WALs/61c2066a8a0f,41615,1732085316699/61c2066a8a0f%2C41615%2C1732085316699.1732085317101
2024-11-20T06:48:37,111 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34847:34847),(127.0.0.1/127.0.0.1:46297:46297)]
2024-11-20T06:48:37,114 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:37,114 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:37,114 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,114 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:37,117 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:48:37,117 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,117 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,117 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:37,118 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:48:37,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,119 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:37,119 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:37,120 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:48:37,120 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,120 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:37,120 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:48:37,121 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:48:37,122 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,122 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:37,122 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,123 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,123 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,125 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,125 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,125 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:48:37,127 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:48:37,131 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:37,131 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794595, jitterRate=0.010380730032920837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:48:37,132 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085317114Initializing all the Stores at 1732085317115 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317115Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085317115Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085317115Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085317115Cleaning up temporary data from old regions at 1732085317125 (+10 ms)Region opened successfully at 1732085317132 (+7 ms)
2024-11-20T06:48:37,132 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:48:37,135 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6842f756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:48:37,136 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:48:37,136 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:48:37,136 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:48:37,136 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:48:37,137 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:48:37,137 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:48:37,137 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:48:37,139 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:48:37,140 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:48:37,167 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:48:37,168 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:48:37,169 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:48:37,176 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:48:37,177 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:48:37,178 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:48:37,184 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:48:37,185 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:48:37,192 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:48:37,194 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:48:37,201 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:48:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:48:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,210 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,41615,1732085316699, sessionid=0x101584a0f100000, setting cluster-up flag (Was=false)
2024-11-20T06:48:37,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,251 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:48:37,252 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,293 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:48:37,294 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:37,295 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:48:37,296 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:37,297 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:48:37,297 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:48:37,297 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,41615,1732085316699
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:48:37,298 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:37,298 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:37,298 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:37,298 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:48:37,299 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:48:37,299 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,299 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:48:37,299 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,299 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085347299
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:48:37,300 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:48:37,300 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:48:37,300 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:48:37,301 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085317301,5,FailOnTimeoutGroup]
2024-11-20T06:48:37,301 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085317301,5,FailOnTimeoutGroup]
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,301 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,301 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,301 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:48:37,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:48:37,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:48:37,308 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:48:37,308 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c
2024-11-20T06:48:37,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:48:37,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:48:37,315 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:37,316 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:48:37,317 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:48:37,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,318 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,318 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:48:37,320 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:48:37,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,320 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,320 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:48:37,322 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:48:37,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,322 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,322 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:48:37,324 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:48:37,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,324 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:48:37,325 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740
2024-11-20T06:48:37,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740
2024-11-20T06:48:37,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:48:37,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:48:37,328 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:48:37,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:37,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:48:37,331 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:37,332 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=861406, jitterRate=0.09533455967903137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:48:37,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085317315Initializing all the Stores at 1732085317316 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317316Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317316Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085317316Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317316Cleaning up temporary data from old regions at 1732085317327 (+11 ms)Region opened successfully at 1732085317332 (+5 ms)
2024-11-20T06:48:37,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:48:37,332 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:48:37,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:48:37,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:48:37,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:48:37,333 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:48:37,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085317332Disabling compacts and flushes for region at 1732085317332Disabling writes for close at 1732085317332Writing region close event to WAL at 1732085317333 (+1 ms)Closed at 1732085317333
2024-11-20T06:48:37,334 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:37,334 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:48:37,334 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:48:37,336 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:48:37,337 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:48:37,381 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(746): ClusterId : 8ee19e05-3c5a-43c3-9988-647353340197
2024-11-20T06:48:37,381 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:48:37,402 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:48:37,402 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:48:37,410 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:48:37,410 DEBUG [RS:0;61c2066a8a0f:41431 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50eca289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:48:37,422 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:41431
2024-11-20T06:48:37,422 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:48:37,422 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:48:37,422 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:48:37,423 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,41615,1732085316699 with port=41431, startcode=1732085316845
2024-11-20T06:48:37,423 DEBUG [RS:0;61c2066a8a0f:41431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:48:37,425 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46951, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:48:37,426 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41615 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,426 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41615 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,427 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c
2024-11-20T06:48:37,427 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38723
2024-11-20T06:48:37,427 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:48:37,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:48:37,435 DEBUG [RS:0;61c2066a8a0f:41431 {}] zookeeper.ZKUtil(111): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,435 WARN  [RS:0;61c2066a8a0f:41431 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:48:37,435 INFO  [RS:0;61c2066a8a0f:41431 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:37,435 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,41431,1732085316845]
2024-11-20T06:48:37,435 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,438 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:48:37,440 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:48:37,440 INFO  [RS:0;61c2066a8a0f:41431 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:48:37,440 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,440 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:48:37,441 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:48:37,441 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,441 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:48:37,442 DEBUG [RS:0;61c2066a8a0f:41431 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,443 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41431,1732085316845-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:48:37,459 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:48:37,459 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41431,1732085316845-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,460 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,460 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.Replication(171): 61c2066a8a0f,41431,1732085316845 started
2024-11-20T06:48:37,475 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:37,475 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,41431,1732085316845, RpcServer on 61c2066a8a0f/172.17.0.2:41431, sessionid=0x101584a0f100001
2024-11-20T06:48:37,476 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:48:37,476 DEBUG [RS:0;61c2066a8a0f:41431 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,476 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41431,1732085316845'
2024-11-20T06:48:37,476 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:48:37,476 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,41431,1732085316845'
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:48:37,477 DEBUG [RS:0;61c2066a8a0f:41431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:48:37,477 INFO  [RS:0;61c2066a8a0f:41431 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:48:37,477 INFO  [RS:0;61c2066a8a0f:41431 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:48:37,487 WARN  [61c2066a8a0f:41615 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:48:37,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:37,580 INFO  [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41431%2C1732085316845, suffix=, logDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845, archiveDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs, maxLogs=32
2024-11-20T06:48:37,580 INFO  [RS:0;61c2066a8a0f:41431 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41431%2C1732085316845.1732085317580
2024-11-20T06:48:37,589 INFO  [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085317580
2024-11-20T06:48:37,603 DEBUG [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34847:34847),(127.0.0.1/127.0.0.1:46297:46297)]
2024-11-20T06:48:37,738 DEBUG [61c2066a8a0f:41615 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:48:37,738 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:37,740 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,41431,1732085316845, state=OPENING
2024-11-20T06:48:37,756 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:48:37,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:48:37,810 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:48:37,810 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:37,810 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:37,810 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41431,1732085316845}]
2024-11-20T06:48:37,964 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:48:37,967 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33021, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:48:37,971 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:48:37,972 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:48:37,974 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C41431%2C1732085316845.meta, suffix=.meta, logDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845, archiveDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs, maxLogs=32
2024-11-20T06:48:37,975 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41431%2C1732085316845.meta.1732085317974.meta
2024-11-20T06:48:37,980 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.meta.1732085317974.meta
2024-11-20T06:48:37,982 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34847:34847),(127.0.0.1/127.0.0.1:46297:46297)]
2024-11-20T06:48:37,983 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:37,983 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:48:37,984 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:48:37,984 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:48:37,984 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:48:37,984 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:37,984 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:48:37,984 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:48:37,986 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:48:37,987 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:48:37,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,987 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,987 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:48:37,988 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:48:37,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,989 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,989 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:48:37,989 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:48:37,989 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,990 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,990 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:48:37,991 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:48:37,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:37,991 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:48:37,991 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:48:37,992 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740
2024-11-20T06:48:37,993 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740
2024-11-20T06:48:37,995 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:48:37,995 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:48:37,995 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:48:37,997 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:48:37,998 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=755855, jitterRate=-0.038881078362464905}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:48:37,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:48:37,999 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085317984Writing region info on filesystem at 1732085317984Initializing all the Stores at 1732085317985 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317985Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317985Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085317986 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085317986Cleaning up temporary data from old regions at 1732085317995 (+9 ms)Running coprocessor post-open hooks at 1732085317998 (+3 ms)Region opened successfully at 1732085317999 (+1 ms)
2024-11-20T06:48:38,000 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085317963
2024-11-20T06:48:38,003 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:48:38,003 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:48:38,004 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:38,005 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,41431,1732085316845, state=OPEN
2024-11-20T06:48:38,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:48:38,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:48:38,062 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:38,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:38,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:48:38,067 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:48:38,067 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,41431,1732085316845 in 252 msec
2024-11-20T06:48:38,071 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:48:38,071 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 733 msec
2024-11-20T06:48:38,072 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:48:38,072 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:48:38,074 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:48:38,074 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,41431,1732085316845, seqNum=-1]
2024-11-20T06:48:38,074 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:48:38,076 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:48:38,083 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 785 msec
2024-11-20T06:48:38,083 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085318083, completionTime=-1
2024-11-20T06:48:38,083 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:48:38,083 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:48:38,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:48:38,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085378085
2024-11-20T06:48:38,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085438085
2024-11-20T06:48:38,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-20T06:48:38,085 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:41615, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,086 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,088 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:48:38,089 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.063sec
2024-11-20T06:48:38,089 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:48:38,089 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:48:38,090 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:48:38,090 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:48:38,090 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:48:38,090 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:48:38,090 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:48:38,092 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:48:38,092 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:48:38,092 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,41615,1732085316699-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:48:38,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3764e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:38,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,41615,-1 for getting cluster id
2024-11-20T06:48:38,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:48:38,184 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8ee19e05-3c5a-43c3-9988-647353340197'
2024-11-20T06:48:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:48:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8ee19e05-3c5a-43c3-9988-647353340197"

2024-11-20T06:48:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a38e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,41615,-1]
2024-11-20T06:48:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:48:38,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:48:38,188 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:48:38,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6800c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:48:38,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:48:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,41431,1732085316845, seqNum=-1]
2024-11-20T06:48:38,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:48:38,192 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:48:38,194 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:38,195 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:48:38,197 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:48:38,197 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T06:48:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,41615,1732085316699
2024-11-20T06:48:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e81af4c
2024-11-20T06:48:38,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T06:48:38,200 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T06:48:38,200 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T06:48:38,200 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T06:48:38,201 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:48:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:38,203 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T06:48:38,204 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:38,204 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-20T06:48:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:48:38,205 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T06:48:38,216 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741835_1011 (size=405)
2024-11-20T06:48:38,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741835_1011 (size=405)
2024-11-20T06:48:38,219 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 40cfdfb1a566a4805d7ea5401d134ab6, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c
2024-11-20T06:48:38,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741836_1012 (size=88)
2024-11-20T06:48:38,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741836_1012 (size=88)
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 40cfdfb1a566a4805d7ea5401d134ab6, disabling compactions & flushes
2024-11-20T06:48:38,228 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6. after waiting 0 ms
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,228 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,228 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 40cfdfb1a566a4805d7ea5401d134ab6:
Waiting for close lock at 1732085318228Disabling compacts and flushes for region at 1732085318228Disabling writes for close at 1732085318228Writing region close event to WAL at 1732085318228Closed at 1732085318228
2024-11-20T06:48:38,229 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T06:48:38,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732085318229"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085318229"}]},"ts":"1732085318229"}
2024-11-20T06:48:38,232 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T06:48:38,233 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T06:48:38,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085318233"}]},"ts":"1732085318233"}
2024-11-20T06:48:38,236 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-20T06:48:38,236 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=40cfdfb1a566a4805d7ea5401d134ab6, ASSIGN}]
2024-11-20T06:48:38,237 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=40cfdfb1a566a4805d7ea5401d134ab6, ASSIGN
2024-11-20T06:48:38,239 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=40cfdfb1a566a4805d7ea5401d134ab6, ASSIGN; state=OFFLINE, location=61c2066a8a0f,41431,1732085316845; forceNewPlan=false, retain=false
2024-11-20T06:48:38,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:38,389 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40cfdfb1a566a4805d7ea5401d134ab6, regionState=OPENING, regionLocation=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:38,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=40cfdfb1a566a4805d7ea5401d134ab6, ASSIGN because future has completed
2024-11-20T06:48:38,392 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40cfdfb1a566a4805d7ea5401d134ab6, server=61c2066a8a0f,41431,1732085316845}]
2024-11-20T06:48:38,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:38,549 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,550 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 40cfdfb1a566a4805d7ea5401d134ab6, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:48:38,550 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,550 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:48:38,550 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,550 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,552 INFO  [StoreOpener-40cfdfb1a566a4805d7ea5401d134ab6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 40cfdfb1a566a4805d7ea5401d134ab6 
2024-11-20T06:48:38,554 INFO  [StoreOpener-40cfdfb1a566a4805d7ea5401d134ab6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40cfdfb1a566a4805d7ea5401d134ab6 columnFamilyName info
2024-11-20T06:48:38,554 DEBUG [StoreOpener-40cfdfb1a566a4805d7ea5401d134ab6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:48:38,555 INFO  [StoreOpener-40cfdfb1a566a4805d7ea5401d134ab6-1 {}] regionserver.HStore(327): Store=40cfdfb1a566a4805d7ea5401d134ab6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:48:38,555 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,556 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,556 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,557 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,557 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,559 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,561 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:48:38,561 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 40cfdfb1a566a4805d7ea5401d134ab6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874586, jitterRate=0.11209404468536377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:48:38,562 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:48:38,562 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 40cfdfb1a566a4805d7ea5401d134ab6:
Running coprocessor pre-open hook at 1732085318550Writing region info on filesystem at 1732085318550Initializing all the Stores at 1732085318552 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085318552Cleaning up temporary data from old regions at 1732085318557 (+5 ms)Running coprocessor post-open hooks at 1732085318562 (+5 ms)Region opened successfully at 1732085318562
2024-11-20T06:48:38,563 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6., pid=6, masterSystemTime=1732085318545
2024-11-20T06:48:38,566 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,566 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:38,567 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=40cfdfb1a566a4805d7ea5401d134ab6, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,41431,1732085316845
2024-11-20T06:48:38,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 40cfdfb1a566a4805d7ea5401d134ab6, server=61c2066a8a0f,41431,1732085316845 because future has completed
2024-11-20T06:48:38,575 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T06:48:38,575 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 40cfdfb1a566a4805d7ea5401d134ab6, server=61c2066a8a0f,41431,1732085316845 in 180 msec
2024-11-20T06:48:38,577 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T06:48:38,577 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=40cfdfb1a566a4805d7ea5401d134ab6, ASSIGN in 339 msec
2024-11-20T06:48:38,578 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T06:48:38,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085318578"}]},"ts":"1732085318578"}
2024-11-20T06:48:38,581 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-20T06:48:38,582 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T06:48:38,584 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 381 msec
2024-11-20T06:48:38,720 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:48:38,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,722 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,722 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,724 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,746 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,751 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:38,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:39,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:39,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:40,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:40,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:41,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:41,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:42,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:42,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:43,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:48:43,217 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-20T06:48:43,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:48:43,218 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-20T06:48:43,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-20T06:48:43,218 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-20T06:48:43,219 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:43,219 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T06:48:43,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:43,439 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:48:43,439 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-20T06:48:43,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:44,052 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:48:44,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,055 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,086 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,086 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,092 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,093 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,093 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,096 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:48:44,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:44,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:45,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:45,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:46,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:46,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:47,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:47,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:48:48,241 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T06:48:48,241 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-20T06:48:48,245 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:48,245 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6., hostname=61c2066a8a0f,41431,1732085316845, seqNum=2]
2024-11-20T06:48:48,256 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:48,262 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T06:48:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-20T06:48:48,263 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T06:48:48,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T06:48:48,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:48,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41431 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-20T06:48:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:48,425 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 40cfdfb1a566a4805d7ea5401d134ab6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T06:48:48,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/42dcad77420c45809e9b9ad5fa521d2f is 1080, key is row0001/info:/1732085328250/Put/seqid=0
2024-11-20T06:48:48,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741837_1013 (size=6033)
2024-11-20T06:48:48,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741837_1013 (size=6033)
2024-11-20T06:48:48,449 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/42dcad77420c45809e9b9ad5fa521d2f
2024-11-20T06:48:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/42dcad77420c45809e9b9ad5fa521d2f as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f
2024-11-20T06:48:48,462 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f, entries=1, sequenceid=5, filesize=5.9 K
2024-11-20T06:48:48,463 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 38ms, sequenceid=5, compaction requested=false
2024-11-20T06:48:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 40cfdfb1a566a4805d7ea5401d134ab6:

2024-11-20T06:48:48,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:48,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-20T06:48:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-20T06:48:48,472 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-20T06:48:48,472 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 203 msec
2024-11-20T06:48:48,474 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 214 msec
2024-11-20T06:48:48,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:49,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:49,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:50,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:50,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:51,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:51,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:52,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:52,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:53,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:53,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:54,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:54,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta after 68063ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:48:54,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:55,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:55,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:56,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:56,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:57,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:57,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-20T06:48:58,301 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T06:48:58,304 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:48:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-20T06:48:58,307 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T06:48:58,309 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T06:48:58,309 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T06:48:58,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:58,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41431 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-20T06:48:58,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:58,463 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 40cfdfb1a566a4805d7ea5401d134ab6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T06:48:58,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/83f236aae93d4fb182d3c5dc69b6f3c1 is 1080, key is row0002/info:/1732085338302/Put/seqid=0
2024-11-20T06:48:58,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741838_1014 (size=6033)
2024-11-20T06:48:58,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741838_1014 (size=6033)
2024-11-20T06:48:58,476 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/83f236aae93d4fb182d3c5dc69b6f3c1
2024-11-20T06:48:58,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/83f236aae93d4fb182d3c5dc69b6f3c1 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1
2024-11-20T06:48:58,489 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1, entries=1, sequenceid=9, filesize=5.9 K
2024-11-20T06:48:58,490 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 27ms, sequenceid=9, compaction requested=false
2024-11-20T06:48:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 40cfdfb1a566a4805d7ea5401d134ab6:

2024-11-20T06:48:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:48:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-20T06:48:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-20T06:48:58,495 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-20T06:48:58,495 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 183 msec
2024-11-20T06:48:58,498 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 192 msec
2024-11-20T06:48:58,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:59,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:59,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:48:59,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723 after 68048ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-20T06:49:00,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:00,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:01,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:01,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:02,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:02,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:03,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:03,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:04,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:04,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:05,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:05,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:06,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:06,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:06,681 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:49:07,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:07,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-20T06:49:08,342 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T06:49:08,345 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41431%2C1732085316845.1732085348345
2024-11-20T06:49:08,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:08,353 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:08,353 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:08,353 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:08,353 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:08,353 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:08,353 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085317580 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085348345
2024-11-20T06:49:08,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741833_1009 (size=5546)
2024-11-20T06:49:08,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741833_1009 (size=5546)
2024-11-20T06:49:08,356 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46297:46297),(127.0.0.1/127.0.0.1:34847:34847)]
2024-11-20T06:49:08,356 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085317580 is not closed yet, will try archiving it next time
2024-11-20T06:49:08,357 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:49:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:49:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-20T06:49:08,359 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T06:49:08,360 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T06:49:08,360 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T06:49:08,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41431 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-20T06:49:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:08,515 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 40cfdfb1a566a4805d7ea5401d134ab6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T06:49:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/3376f9240f6c4ba09d39ab8cba483ea8 is 1080, key is row0003/info:/1732085348343/Put/seqid=0
2024-11-20T06:49:08,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741840_1016 (size=6033)
2024-11-20T06:49:08,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741840_1016 (size=6033)
2024-11-20T06:49:08,529 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/3376f9240f6c4ba09d39ab8cba483ea8
2024-11-20T06:49:08,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:08,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/3376f9240f6c4ba09d39ab8cba483ea8 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8
2024-11-20T06:49:08,545 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8, entries=1, sequenceid=13, filesize=5.9 K
2024-11-20T06:49:08,546 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 30ms, sequenceid=13, compaction requested=true
2024-11-20T06:49:08,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 40cfdfb1a566a4805d7ea5401d134ab6:

2024-11-20T06:49:08,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:08,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-20T06:49:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-20T06:49:08,550 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-20T06:49:08,550 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 187 msec
2024-11-20T06:49:08,553 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 194 msec
2024-11-20T06:49:09,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:09,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:10,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:10,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:11,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:11,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:12,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:12,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:13,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:13,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:14,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:14,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:15,353 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:15,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:16,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:16,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:17,354 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:17,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:18,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:18,450 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T06:49:18,450 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T06:49:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-20T06:49:18,461 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T06:49:18,461 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:18,463 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:49:18,463 DEBUG [Time-limited test {}] regionserver.HStore(1541): 40cfdfb1a566a4805d7ea5401d134ab6/info is initiating minor compaction (all files)
2024-11-20T06:49:18,463 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:49:18,463 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:18,463 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 40cfdfb1a566a4805d7ea5401d134ab6/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:18,464 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8] into tmpdir=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp, totalSize=17.7 K
2024-11-20T06:49:18,464 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 42dcad77420c45809e9b9ad5fa521d2f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732085328250
2024-11-20T06:49:18,465 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 83f236aae93d4fb182d3c5dc69b6f3c1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732085338302
2024-11-20T06:49:18,465 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3376f9240f6c4ba09d39ab8cba483ea8, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732085348343
2024-11-20T06:49:18,480 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 40cfdfb1a566a4805d7ea5401d134ab6#info#compaction#46 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:18,480 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/33d00fe76bfd4cc1a047ab7070d8fc5f is 1080, key is row0001/info:/1732085328250/Put/seqid=0
2024-11-20T06:49:18,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741841_1017 (size=8296)
2024-11-20T06:49:18,493 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741841_1017 (size=8296)
2024-11-20T06:49:18,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/33d00fe76bfd4cc1a047ab7070d8fc5f as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/33d00fe76bfd4cc1a047ab7070d8fc5f
2024-11-20T06:49:18,509 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 40cfdfb1a566a4805d7ea5401d134ab6/info of 40cfdfb1a566a4805d7ea5401d134ab6 into 33d00fe76bfd4cc1a047ab7070d8fc5f(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:18,509 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 40cfdfb1a566a4805d7ea5401d134ab6:

2024-11-20T06:49:18,512 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41431%2C1732085316845.1732085358512
2024-11-20T06:49:18,518 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:18,519 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:18,519 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:18,519 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:18,519 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:18,519 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085348345 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085358512
2024-11-20T06:49:18,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741839_1015 (size=2520)
2024-11-20T06:49:18,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741839_1015 (size=2520)
2024-11-20T06:49:18,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46297:46297),(127.0.0.1/127.0.0.1:34847:34847)]
2024-11-20T06:49:18,523 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085317580 to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs/61c2066a8a0f%2C41431%2C1732085316845.1732085317580
2024-11-20T06:49:18,524 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:49:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:49:18,526 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-20T06:49:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-20T06:49:18,527 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-20T06:49:18,527 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-20T06:49:18,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:18,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41431 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-20T06:49:18,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:18,681 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 40cfdfb1a566a4805d7ea5401d134ab6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T06:49:18,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/b0f1af1fed0e452cacee07dc9820bc0e is 1080, key is row0000/info:/1732085358510/Put/seqid=0
2024-11-20T06:49:18,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741843_1019 (size=6033)
2024-11-20T06:49:18,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741843_1019 (size=6033)
2024-11-20T06:49:19,094 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/b0f1af1fed0e452cacee07dc9820bc0e
2024-11-20T06:49:19,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/b0f1af1fed0e452cacee07dc9820bc0e as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/b0f1af1fed0e452cacee07dc9820bc0e
2024-11-20T06:49:19,106 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/b0f1af1fed0e452cacee07dc9820bc0e, entries=1, sequenceid=18, filesize=5.9 K
2024-11-20T06:49:19,108 INFO  [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 426ms, sequenceid=18, compaction requested=false
2024-11-20T06:49:19,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 40cfdfb1a566a4805d7ea5401d134ab6:

2024-11-20T06:49:19,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:19,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-20T06:49:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-20T06:49:19,112 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-20T06:49:19,112 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 583 msec
2024-11-20T06:49:19,115 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 589 msec
2024-11-20T06:49:19,356 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:19,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:20,357 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:20,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:21,358 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:21,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:22,359 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:22,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:23,360 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:23,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:23,550 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 40cfdfb1a566a4805d7ea5401d134ab6, had cached 0 bytes from a total of 14329
2024-11-20T06:49:24,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:24,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:25,361 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:25,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:26,362 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:26,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:27,363 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:27,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:28,364 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:28,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41615 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-20T06:49:28,551 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-20T06:49:28,553 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C41431%2C1732085316845.1732085368553
2024-11-20T06:49:28,559 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,560 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,560 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,560 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,560 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,560 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085358512 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085368553
2024-11-20T06:49:28,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34847:34847),(127.0.0.1/127.0.0.1:46297:46297)]
2024-11-20T06:49:28,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085358512 is not closed yet, will try archiving it next time
2024-11-20T06:49:28,561 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/WALs/61c2066a8a0f,41431,1732085316845/61c2066a8a0f%2C41431%2C1732085316845.1732085348345 to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs/61c2066a8a0f%2C41431%2C1732085316845.1732085348345
2024-11-20T06:49:28,561 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:49:28,562 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:49:28,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741842_1018 (size=2026)
2024-11-20T06:49:28,562 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:49:28,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:28,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:28,562 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:49:28,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741842_1018 (size=2026)
2024-11-20T06:49:28,565 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:49:28,565 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1737499502, stopped=false
2024-11-20T06:49:28,565 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,41615,1732085316699
2024-11-20T06:49:28,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:49:28,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:49:28,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:28,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:28,590 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:49:28,590 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:49:28,590 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:49:28,590 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:28,591 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,41431,1732085316845' *****
2024-11-20T06:49:28,591 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:49:28,591 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:49:28,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:49:28,591 INFO  [RS:0;61c2066a8a0f:41431 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:49:28,591 INFO  [RS:0;61c2066a8a0f:41431 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:49:28,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:49:28,591 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(3091): Received CLOSE for 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:49:28,592 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,41431,1732085316845
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:41431.
2024-11-20T06:49:28,592 DEBUG [RS:0;61c2066a8a0f:41431 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:49:28,592 DEBUG [RS:0;61c2066a8a0f:41431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:49:28,592 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 40cfdfb1a566a4805d7ea5401d134ab6, disabling compactions & flushes
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:49:28,592 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:28,592 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:28,592 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6. after waiting 0 ms
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:28,593 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 40cfdfb1a566a4805d7ea5401d134ab6 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-20T06:49:28,593 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-20T06:49:28,593 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 40cfdfb1a566a4805d7ea5401d134ab6=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.}
2024-11-20T06:49:28,593 DEBUG [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 40cfdfb1a566a4805d7ea5401d134ab6
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:49:28,593 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:49:28,593 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:49:28,593 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-20T06:49:28,599 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/ad8623fd88c64c5da8afa08ac46f85b3 is 1080, key is row0001/info:/1732085368552/Put/seqid=0
2024-11-20T06:49:28,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741845_1021 (size=6033)
2024-11-20T06:49:28,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741845_1021 (size=6033)
2024-11-20T06:49:28,604 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/ad8623fd88c64c5da8afa08ac46f85b3
2024-11-20T06:49:28,612 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/.tmp/info/ad8623fd88c64c5da8afa08ac46f85b3 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/ad8623fd88c64c5da8afa08ac46f85b3
2024-11-20T06:49:28,614 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/info/b1951540f0b4481da2888e8af934c67f is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6./info:regioninfo/1732085318567/Put/seqid=0
2024-11-20T06:49:28,617 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/ad8623fd88c64c5da8afa08ac46f85b3, entries=1, sequenceid=22, filesize=5.9 K
2024-11-20T06:49:28,619 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 25ms, sequenceid=22, compaction requested=true
2024-11-20T06:49:28,619 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8] to archive
2024-11-20T06:49:28,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741846_1022 (size=7308)
2024-11-20T06:49:28,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741846_1022 (size=7308)
2024-11-20T06:49:28,620 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:49:28,620 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/info/b1951540f0b4481da2888e8af934c67f
2024-11-20T06:49:28,622 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/42dcad77420c45809e9b9ad5fa521d2f
2024-11-20T06:49:28,623 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1 to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/83f236aae93d4fb182d3c5dc69b6f3c1
2024-11-20T06:49:28,624 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8 to hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/info/3376f9240f6c4ba09d39ab8cba483ea8
2024-11-20T06:49:28,625 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=61c2066a8a0f:41615 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-20T06:49:28,625 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [42dcad77420c45809e9b9ad5fa521d2f=6033, 83f236aae93d4fb182d3c5dc69b6f3c1=6033, 3376f9240f6c4ba09d39ab8cba483ea8=6033]
2024-11-20T06:49:28,628 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/40cfdfb1a566a4805d7ea5401d134ab6/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-20T06:49:28,629 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:28,629 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 40cfdfb1a566a4805d7ea5401d134ab6:
Waiting for close lock at 1732085368592Running coprocessor pre-close hooks at 1732085368592Disabling compacts and flushes for region at 1732085368592Disabling writes for close at 1732085368593 (+1 ms)Obtaining lock to block concurrent updates at 1732085368593Preparing flush snapshotting stores in 40cfdfb1a566a4805d7ea5401d134ab6 at 1732085368593Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732085368593Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6. at 1732085368594 (+1 ms)Flushing 40cfdfb1a566a4805d7ea5401d134ab6/info: creating writer at 1732085368594Flushing 40cfdfb1a566a4805d7ea5401d134ab6/info: appending metadata at 1732085368598 (+4 ms)Flushing 40cfdfb1a566a4805d7ea5401d134ab6/info: closing flushed file at 1732085368598Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9e8fea6: reopening flushed file at 1732085368611 (+13 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 40cfdfb1a566a4805d7ea5401d134ab6 in 25ms, sequenceid=22, compaction requested=true at 1732085368619 (+8 ms)Writing region close event to WAL at 1732085368626 (+7 ms)Running coprocessor post-close hooks at 1732085368629 (+3 ms)Closed at 1732085368629
2024-11-20T06:49:28,629 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732085318200.40cfdfb1a566a4805d7ea5401d134ab6.
2024-11-20T06:49:28,641 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/ns/22fc663e94c94427882719099ae98f32 is 43, key is default/ns:d/1732085318077/Put/seqid=0
2024-11-20T06:49:28,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741847_1023 (size=5153)
2024-11-20T06:49:28,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741847_1023 (size=5153)
2024-11-20T06:49:28,646 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/ns/22fc663e94c94427882719099ae98f32
2024-11-20T06:49:28,667 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/table/005a95789eba485385f3ca32713e5d8c is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732085318578/Put/seqid=0
2024-11-20T06:49:28,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741848_1024 (size=5508)
2024-11-20T06:49:28,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741848_1024 (size=5508)
2024-11-20T06:49:28,672 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/table/005a95789eba485385f3ca32713e5d8c
2024-11-20T06:49:28,677 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/info/b1951540f0b4481da2888e8af934c67f as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/info/b1951540f0b4481da2888e8af934c67f
2024-11-20T06:49:28,683 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/info/b1951540f0b4481da2888e8af934c67f, entries=10, sequenceid=11, filesize=7.1 K
2024-11-20T06:49:28,684 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/ns/22fc663e94c94427882719099ae98f32 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/ns/22fc663e94c94427882719099ae98f32
2024-11-20T06:49:28,689 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/ns/22fc663e94c94427882719099ae98f32, entries=2, sequenceid=11, filesize=5.0 K
2024-11-20T06:49:28,691 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/.tmp/table/005a95789eba485385f3ca32713e5d8c as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/table/005a95789eba485385f3ca32713e5d8c
2024-11-20T06:49:28,697 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/table/005a95789eba485385f3ca32713e5d8c, entries=2, sequenceid=11, filesize=5.4 K
2024-11-20T06:49:28,698 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false
2024-11-20T06:49:28,706 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-20T06:49:28,707 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:49:28,707 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:49:28,707 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085368593Running coprocessor pre-close hooks at 1732085368593Disabling compacts and flushes for region at 1732085368593Disabling writes for close at 1732085368593Obtaining lock to block concurrent updates at 1732085368593Preparing flush snapshotting stores in 1588230740 at 1732085368593Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732085368594 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085368595 (+1 ms)Flushing 1588230740/info: creating writer at 1732085368595Flushing 1588230740/info: appending metadata at 1732085368614 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732085368614Flushing 1588230740/ns: creating writer at 1732085368626 (+12 ms)Flushing 1588230740/ns: appending metadata at 1732085368641 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732085368641Flushing 1588230740/table: creating writer at 1732085368652 (+11 ms)Flushing 1588230740/table: appending metadata at 1732085368666 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732085368666Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c4a2e9a: reopening flushed file at 1732085368677 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b1d9ef5: reopening flushed file at 1732085368683 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@384fa38f: reopening flushed file at 1732085368690 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false at 1732085368698 (+8 ms)Writing region close event to WAL at 1732085368702 (+4 ms)Running coprocessor post-close hooks at 1732085368707 (+5 ms)Closed at 1732085368707
2024-11-20T06:49:28,707 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:49:28,793 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,41431,1732085316845; all regions closed.
2024-11-20T06:49:28,794 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,794 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,794 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,794 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,794 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741834_1010 (size=3306)
2024-11-20T06:49:28,796 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741834_1010 (size=3306)
2024-11-20T06:49:28,798 DEBUG [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs
2024-11-20T06:49:28,798 INFO  [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C41431%2C1732085316845.meta:.meta(num 1732085317974)
2024-11-20T06:49:28,799 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,799 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,799 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,799 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,799 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741844_1020 (size=1252)
2024-11-20T06:49:28,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741844_1020 (size=1252)
2024-11-20T06:49:28,804 DEBUG [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/oldWALs
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C41431%2C1732085316845:(num 1732085368553)
2024-11-20T06:49:28,804 DEBUG [RS:0;61c2066a8a0f:41431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:49:28,804 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:49:28,804 INFO  [RS:0;61c2066a8a0f:41431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41431
2024-11-20T06:49:28,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:49:28,817 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:49:28,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,41431,1732085316845
2024-11-20T06:49:28,825 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,41431,1732085316845]
2024-11-20T06:49:28,833 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,41431,1732085316845 already deleted, retry=false
2024-11-20T06:49:28,833 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,41431,1732085316845 expired; onlineServers=0
2024-11-20T06:49:28,833 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,41615,1732085316699' *****
2024-11-20T06:49:28,833 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:49:28,833 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:49:28,833 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:49:28,833 DEBUG [M:0;61c2066a8a0f:41615 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:49:28,833 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:49:28,833 DEBUG [M:0;61c2066a8a0f:41615 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:49:28,833 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085317301 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085317301,5,FailOnTimeoutGroup]
2024-11-20T06:49:28,833 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085317301 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085317301,5,FailOnTimeoutGroup]
2024-11-20T06:49:28,834 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:49:28,834 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:49:28,834 DEBUG [M:0;61c2066a8a0f:41615 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:49:28,834 INFO  [M:0;61c2066a8a0f:41615 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:49:28,834 INFO  [M:0;61c2066a8a0f:41615 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:49:28,834 INFO  [M:0;61c2066a8a0f:41615 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:49:28,834 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:49:28,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:49:28,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:28,842 DEBUG [M:0;61c2066a8a0f:41615 {}] zookeeper.ZKUtil(347): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:49:28,842 WARN  [M:0;61c2066a8a0f:41615 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:49:28,842 INFO  [M:0;61c2066a8a0f:41615 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/.lastflushedseqids
2024-11-20T06:49:28,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741849_1025 (size=130)
2024-11-20T06:49:28,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741849_1025 (size=130)
2024-11-20T06:49:28,848 INFO  [M:0;61c2066a8a0f:41615 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:49:28,848 INFO  [M:0;61c2066a8a0f:41615 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:49:28,848 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:49:28,848 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:28,848 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:28,848 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:49:28,848 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:28,849 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.53 KB heapSize=54.91 KB
2024-11-20T06:49:28,867 DEBUG [M:0;61c2066a8a0f:41615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c204c197d3ae44d48a2f2106b4e2acd6 is 82, key is hbase:meta,,1/info:regioninfo/1732085318004/Put/seqid=0
2024-11-20T06:49:28,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741850_1026 (size=5672)
2024-11-20T06:49:28,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741850_1026 (size=5672)
2024-11-20T06:49:28,872 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c204c197d3ae44d48a2f2106b4e2acd6
2024-11-20T06:49:28,894 DEBUG [M:0;61c2066a8a0f:41615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e0212afd58248a9ae4cf5225e7df90f is 796, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085318583/Put/seqid=0
2024-11-20T06:49:28,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741851_1027 (size=7817)
2024-11-20T06:49:28,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741851_1027 (size=7817)
2024-11-20T06:49:28,900 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.93 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e0212afd58248a9ae4cf5225e7df90f
2024-11-20T06:49:28,906 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e0212afd58248a9ae4cf5225e7df90f
2024-11-20T06:49:28,922 DEBUG [M:0;61c2066a8a0f:41615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4de7be5eec57431c981154cc4fda413e is 69, key is 61c2066a8a0f,41431,1732085316845/rs:state/1732085317426/Put/seqid=0
2024-11-20T06:49:28,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:49:28,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41431-0x101584a0f100001, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:49:28,925 INFO  [RS:0;61c2066a8a0f:41431 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:49:28,925 INFO  [RS:0;61c2066a8a0f:41431 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,41431,1732085316845; zookeeper connection closed.
2024-11-20T06:49:28,925 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a24f263 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a24f263
2024-11-20T06:49:28,926 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:49:28,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741852_1028 (size=5156)
2024-11-20T06:49:28,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741852_1028 (size=5156)
2024-11-20T06:49:28,928 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4de7be5eec57431c981154cc4fda413e
2024-11-20T06:49:28,954 DEBUG [M:0;61c2066a8a0f:41615 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ea797da1347f42be8e77d2e4a38da152 is 52, key is load_balancer_on/state:d/1732085318196/Put/seqid=0
2024-11-20T06:49:28,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741853_1029 (size=5056)
2024-11-20T06:49:28,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741853_1029 (size=5056)
2024-11-20T06:49:28,959 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ea797da1347f42be8e77d2e4a38da152
2024-11-20T06:49:28,965 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c204c197d3ae44d48a2f2106b4e2acd6 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c204c197d3ae44d48a2f2106b4e2acd6
2024-11-20T06:49:28,971 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c204c197d3ae44d48a2f2106b4e2acd6, entries=8, sequenceid=121, filesize=5.5 K
2024-11-20T06:49:28,972 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4e0212afd58248a9ae4cf5225e7df90f as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e0212afd58248a9ae4cf5225e7df90f
2024-11-20T06:49:28,977 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4e0212afd58248a9ae4cf5225e7df90f
2024-11-20T06:49:28,977 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4e0212afd58248a9ae4cf5225e7df90f, entries=14, sequenceid=121, filesize=7.6 K
2024-11-20T06:49:28,978 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4de7be5eec57431c981154cc4fda413e as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4de7be5eec57431c981154cc4fda413e
2024-11-20T06:49:28,984 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4de7be5eec57431c981154cc4fda413e, entries=1, sequenceid=121, filesize=5.0 K
2024-11-20T06:49:28,985 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ea797da1347f42be8e77d2e4a38da152 as hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ea797da1347f42be8e77d2e4a38da152
2024-11-20T06:49:28,990 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38723/user/jenkins/test-data/7a2d0ebf-2f45-5457-9746-f355ee03875c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ea797da1347f42be8e77d2e4a38da152, entries=1, sequenceid=121, filesize=4.9 K
2024-11-20T06:49:28,991 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.53 KB/44578, heapSize ~54.84 KB/56160, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=121, compaction requested=false
2024-11-20T06:49:28,992 INFO  [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:28,993 DEBUG [M:0;61c2066a8a0f:41615 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085368848Disabling compacts and flushes for region at 1732085368848Disabling writes for close at 1732085368848Obtaining lock to block concurrent updates at 1732085368849 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085368849Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44578, getHeapSize=56160, getOffHeapSize=0, getCellsCount=140 at 1732085368849Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085368850 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085368850Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085368867 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085368867Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085368877 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085368894 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085368894Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085368907 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085368922 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085368922Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085368933 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085368953 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085368953Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6df68a1c: reopening flushed file at 1732085368965 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c387aad: reopening flushed file at 1732085368971 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@113bf7de: reopening flushed file at 1732085368977 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2353c431: reopening flushed file at 1732085368984 (+7 ms)Finished flush of dataSize ~43.53 KB/44578, heapSize ~54.84 KB/56160, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 142ms, sequenceid=121, compaction requested=false at 1732085368991 (+7 ms)Writing region close event to WAL at 1732085368992 (+1 ms)Closed at 1732085368992
2024-11-20T06:49:28,993 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,993 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,993 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,993 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,993 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:49:28,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41231 is added to blk_1073741830_1006 (size=52975)
2024-11-20T06:49:28,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39143 is added to blk_1073741830_1006 (size=52975)
2024-11-20T06:49:28,996 INFO  [M:0;61c2066a8a0f:41615 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:49:28,996 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:49:28,996 INFO  [M:0;61c2066a8a0f:41615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41615
2024-11-20T06:49:28,996 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:49:29,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:49:29,117 INFO  [M:0;61c2066a8a0f:41615 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:49:29,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41615-0x101584a0f100000, quorum=127.0.0.1:62815, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:49:29,151 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@75e789f0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:49:29,151 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@523d16c3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:49:29,151 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:49:29,152 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@311facd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:49:29,152 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b7fc8f3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,STOPPED}
2024-11-20T06:49:29,154 WARN  [BP-1170907024-172.17.0.2-1732085315144 heartbeating to localhost/127.0.0.1:38723 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:49:29,154 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:49:29,154 WARN  [BP-1170907024-172.17.0.2-1732085315144 heartbeating to localhost/127.0.0.1:38723 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1170907024-172.17.0.2-1732085315144 (Datanode Uuid 05b5a924-6c82-4797-ab45-d67ddabb0bf1) service to localhost/127.0.0.1:38723
2024-11-20T06:49:29,154 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:49:29,155 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data3/current/BP-1170907024-172.17.0.2-1732085315144 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:49:29,155 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data4/current/BP-1170907024-172.17.0.2-1732085315144 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:49:29,155 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:49:29,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c1ed8d4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:49:29,159 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d36967f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:49:29,159 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:49:29,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@507832d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:49:29,159 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b079ea2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,STOPPED}
2024-11-20T06:49:29,160 WARN  [BP-1170907024-172.17.0.2-1732085315144 heartbeating to localhost/127.0.0.1:38723 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:49:29,160 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:49:29,161 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:49:29,161 WARN  [BP-1170907024-172.17.0.2-1732085315144 heartbeating to localhost/127.0.0.1:38723 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1170907024-172.17.0.2-1732085315144 (Datanode Uuid 47cfc7e8-104a-4993-8c7d-95b58c31fd3c) service to localhost/127.0.0.1:38723
2024-11-20T06:49:29,161 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data1/current/BP-1170907024-172.17.0.2-1732085315144 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:49:29,161 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/cluster_86bebe18-00a8-6ec3-e979-88a576475090/data/data2/current/BP-1170907024-172.17.0.2-1732085315144 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:49:29,162 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:49:29,168 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76b2d62a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:49:29,169 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47d3f616{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:49:29,169 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:49:29,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fb33a9d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:49:29,169 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20e4ef1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir/,STOPPED}
2024-11-20T06:49:29,176 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:49:29,199 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:49:29,207 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=207 (was 181)
Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:38723 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:38723
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:38723 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38723
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:38723 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38723
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38723
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38723
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38723
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/61c2066a8a0f:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=277 (was 249) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3576 (was 3769)
2024-11-20T06:49:29,215 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=207, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=3576
2024-11-20T06:49:29,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:49:29,215 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.log.dir so I do NOT create it in target/test-data/9965841b-52ca-dd00-b936-e7541226f10d
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/196d36f8-37aa-a757-4d8b-e6d6c4179598/hadoop.tmp.dir so I do NOT create it in target/test-data/9965841b-52ca-dd00-b936-e7541226f10d
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530, deleteOnExit=true
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/test.cache.data in system properties and HBase conf
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:49:29,216 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:49:29,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:49:29,217 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:49:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:49:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:49:29,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:49:29,234 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:49:29,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:29,446 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:49:29,545 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:49:29,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:29,549 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:49:29,550 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:49:29,550 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:49:29,550 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:49:29,551 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:49:29,551 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e36d39c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:49:29,551 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75966949{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:49:29,654 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7018a0ae{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/java.io.tmpdir/jetty-localhost-42919-hadoop-hdfs-3_4_1-tests_jar-_-any-12903093975940147744/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:49:29,655 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7305dd28{HTTP/1.1, (http/1.1)}{localhost:42919}
2024-11-20T06:49:29,655 INFO  [Time-limited test {}] server.Server(415): Started @252997ms
2024-11-20T06:49:29,667 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:49:29,903 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:49:29,907 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:49:29,907 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:49:29,907 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:49:29,908 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:49:29,908 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bb23947{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:49:29,908 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@60b9b83d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:49:30,013 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@631c133{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/java.io.tmpdir/jetty-localhost-45749-hadoop-hdfs-3_4_1-tests_jar-_-any-4370051931387045961/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:49:30,013 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3523e770{HTTP/1.1, (http/1.1)}{localhost:45749}
2024-11-20T06:49:30,013 INFO  [Time-limited test {}] server.Server(415): Started @253355ms
2024-11-20T06:49:30,015 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:49:30,043 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:49:30,045 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:49:30,046 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:49:30,046 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:49:30,046 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-20T06:49:30,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ab86f9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:49:30,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fc2e7d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:49:30,154 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@dcfcbff{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/java.io.tmpdir/jetty-localhost-42313-hadoop-hdfs-3_4_1-tests_jar-_-any-14678469347825374962/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:49:30,155 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3dc4994c{HTTP/1.1, (http/1.1)}{localhost:42313}
2024-11-20T06:49:30,155 INFO  [Time-limited test {}] server.Server(415): Started @253497ms
2024-11-20T06:49:30,156 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:49:30,365 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:30,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:30,748 WARN  [Thread-1980 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data1/current/BP-101583425-172.17.0.2-1732085369237/current, will proceed with Du for space computation calculation, 
2024-11-20T06:49:30,748 WARN  [Thread-1981 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data2/current/BP-101583425-172.17.0.2-1732085369237/current, will proceed with Du for space computation calculation, 
2024-11-20T06:49:30,767 WARN  [Thread-1944 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:49:30,769 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7dbde2dfc9dad58b with lease ID 0x2c3317c6a315e012: Processing first storage report for DS-b7f6178d-ef63-4de6-8297-6f91d1829271 from datanode DatanodeRegistration(127.0.0.1:37317, datanodeUuid=c2f8cf3e-d0bc-4d65-9190-98b6585122bf, infoPort=44703, infoSecurePort=0, ipcPort=45367, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237)
2024-11-20T06:49:30,769 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7dbde2dfc9dad58b with lease ID 0x2c3317c6a315e012: from storage DS-b7f6178d-ef63-4de6-8297-6f91d1829271 node DatanodeRegistration(127.0.0.1:37317, datanodeUuid=c2f8cf3e-d0bc-4d65-9190-98b6585122bf, infoPort=44703, infoSecurePort=0, ipcPort=45367, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:49:30,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7dbde2dfc9dad58b with lease ID 0x2c3317c6a315e012: Processing first storage report for DS-3dc740ca-c299-4756-965f-6a3982e4b747 from datanode DatanodeRegistration(127.0.0.1:37317, datanodeUuid=c2f8cf3e-d0bc-4d65-9190-98b6585122bf, infoPort=44703, infoSecurePort=0, ipcPort=45367, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237)
2024-11-20T06:49:30,770 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7dbde2dfc9dad58b with lease ID 0x2c3317c6a315e012: from storage DS-3dc740ca-c299-4756-965f-6a3982e4b747 node DatanodeRegistration(127.0.0.1:37317, datanodeUuid=c2f8cf3e-d0bc-4d65-9190-98b6585122bf, infoPort=44703, infoSecurePort=0, ipcPort=45367, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:49:30,848 WARN  [Thread-1991 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data3/current/BP-101583425-172.17.0.2-1732085369237/current, will proceed with Du for space computation calculation, 
2024-11-20T06:49:30,848 WARN  [Thread-1992 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data4/current/BP-101583425-172.17.0.2-1732085369237/current, will proceed with Du for space computation calculation, 
2024-11-20T06:49:30,869 WARN  [Thread-1967 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:49:30,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda2549b6790930bf with lease ID 0x2c3317c6a315e013: Processing first storage report for DS-b7ce9d34-62f7-4b97-b92e-d69c114a3a37 from datanode DatanodeRegistration(127.0.0.1:36161, datanodeUuid=9349f7e6-92c7-423b-9349-9a118e77bcbb, infoPort=40257, infoSecurePort=0, ipcPort=38179, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237)
2024-11-20T06:49:30,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda2549b6790930bf with lease ID 0x2c3317c6a315e013: from storage DS-b7ce9d34-62f7-4b97-b92e-d69c114a3a37 node DatanodeRegistration(127.0.0.1:36161, datanodeUuid=9349f7e6-92c7-423b-9349-9a118e77bcbb, infoPort=40257, infoSecurePort=0, ipcPort=38179, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:49:30,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda2549b6790930bf with lease ID 0x2c3317c6a315e013: Processing first storage report for DS-3312d830-bd27-4df1-90ce-ede34a81558e from datanode DatanodeRegistration(127.0.0.1:36161, datanodeUuid=9349f7e6-92c7-423b-9349-9a118e77bcbb, infoPort=40257, infoSecurePort=0, ipcPort=38179, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237)
2024-11-20T06:49:30,871 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda2549b6790930bf with lease ID 0x2c3317c6a315e013: from storage DS-3312d830-bd27-4df1-90ce-ede34a81558e node DatanodeRegistration(127.0.0.1:36161, datanodeUuid=9349f7e6-92c7-423b-9349-9a118e77bcbb, infoPort=40257, infoSecurePort=0, ipcPort=38179, storageInfo=lv=-57;cid=testClusterID;nsid=1905950178;c=1732085369237), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:49:30,882 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d
2024-11-20T06:49:30,886 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/zookeeper_0, clientPort=57279, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:49:30,886 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57279
2024-11-20T06:49:30,887 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:30,889 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:30,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:49:30,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:49:30,899 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942 with version=8
2024-11-20T06:49:30,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:49:30,902 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:49:30,903 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38673
2024-11-20T06:49:30,905 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38673 connecting to ZooKeeper ensemble=127.0.0.1:57279
2024-11-20T06:49:30,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386730x0, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:49:30,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38673-0x101584ae2c90000 connected
2024-11-20T06:49:31,017 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:31,019 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:31,022 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:49:31,022 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942, hbase.cluster.distributed=false
2024-11-20T06:49:31,024 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:49:31,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38673
2024-11-20T06:49:31,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38673
2024-11-20T06:49:31,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38673
2024-11-20T06:49:31,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38673
2024-11-20T06:49:31,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38673
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:49:31,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:49:31,047 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40669
2024-11-20T06:49:31,048 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40669 connecting to ZooKeeper ensemble=127.0.0.1:57279
2024-11-20T06:49:31,048 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:31,050 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:31,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406690x0, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:49:31,059 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406690x0, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:49:31,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40669-0x101584ae2c90001 connected
2024-11-20T06:49:31,059 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:49:31,059 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:49:31,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:49:31,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:49:31,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-20T06:49:31,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40669
2024-11-20T06:49:31,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40669
2024-11-20T06:49:31,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-20T06:49:31,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40669
2024-11-20T06:49:31,078 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:38673
2024-11-20T06:49:31,078 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:49:31,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:49:31,089 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:49:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,100 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:49:31,101 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,38673,1732085370902 from backup master directory
2024-11-20T06:49:31,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:49:31,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:49:31,108 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:49:31,108 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,113 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/hbase.id] with ID: 8192399d-b065-432f-b184-365d049b3a13
2024-11-20T06:49:31,113 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/.tmp/hbase.id
2024-11-20T06:49:31,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:49:31,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:49:31,120 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/.tmp/hbase.id]:[hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/hbase.id]
2024-11-20T06:49:31,132 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:31,133 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:49:31,134 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T06:49:31,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:49:31,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:49:31,149 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:49:31,150 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:49:31,150 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:49:31,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:49:31,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:49:31,158 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store
2024-11-20T06:49:31,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:49:31,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:49:31,164 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:31,164 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:49:31,164 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:31,164 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:31,164 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:49:31,164 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:31,164 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:49:31,165 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085371164Disabling compacts and flushes for region at 1732085371164Disabling writes for close at 1732085371164Writing region close event to WAL at 1732085371164Closed at 1732085371164
2024-11-20T06:49:31,165 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/.initializing
2024-11-20T06:49:31,165 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/WALs/61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,167 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C38673%2C1732085370902, suffix=, logDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/WALs/61c2066a8a0f,38673,1732085370902, archiveDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/oldWALs, maxLogs=10
2024-11-20T06:49:31,168 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C38673%2C1732085370902.1732085371167
2024-11-20T06:49:31,172 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/WALs/61c2066a8a0f,38673,1732085370902/61c2066a8a0f%2C38673%2C1732085370902.1732085371167
2024-11-20T06:49:31,173 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40257:40257),(127.0.0.1/127.0.0.1:44703:44703)]
2024-11-20T06:49:31,174 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:49:31,174 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:31,174 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,174 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,175 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:49:31,176 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:49:31,177 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,177 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:31,177 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:49:31,178 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:49:31,178 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:31,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:49:31,180 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:49:31,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,180 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:31,180 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:49:31,181 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:49:31,181 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,181 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:31,182 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,182 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,183 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,184 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,184 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,184 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:49:31,185 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:49:31,188 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:49:31,188 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=736211, jitterRate=-0.06386043131351471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:49:31,189 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085371174Initializing all the Stores at 1732085371175 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085371175Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085371175Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085371175Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085371175Cleaning up temporary data from old regions at 1732085371184 (+9 ms)Region opened successfully at 1732085371189 (+5 ms)
2024-11-20T06:49:31,189 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:49:31,192 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb23332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:49:31,193 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:49:31,193 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:49:31,193 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:49:31,194 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:49:31,194 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:49:31,194 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:49:31,194 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:49:31,197 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:49:31,198 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:49:31,208 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:49:31,208 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:49:31,209 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:49:31,216 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:49:31,217 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:49:31,218 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:49:31,225 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:49:31,225 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:49:31,233 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:49:31,236 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:49:31,241 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:49:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:49:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:49:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,250 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,38673,1732085370902, sessionid=0x101584ae2c90000, setting cluster-up flag (Was=false)
2024-11-20T06:49:31,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,292 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:49:31,293 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,341 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:49:31,343 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:31,344 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:49:31,345 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:49:31,346 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:49:31,346 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:49:31,346 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,38673,1732085370902
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:49:31,347 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,348 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:49:31,348 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085401349
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:49:31,349 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:49:31,349 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:49:31,349 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:49:31,350 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,350 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:49:31,350 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:49:31,350 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:49:31,351 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:49:31,351 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:49:31,351 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,351 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:49:31,354 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085371351,5,FailOnTimeoutGroup]
2024-11-20T06:49:31,354 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085371354,5,FailOnTimeoutGroup]
2024-11-20T06:49:31,355 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,355 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:49:31,355 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,355 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:49:31,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:49:31,360 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:49:31,360 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942
2024-11-20T06:49:31,366 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:31,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:49:31,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:49:31,368 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(746): ClusterId : 8192399d-b065-432f-b184-365d049b3a13
2024-11-20T06:49:31,368 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:49:31,372 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:31,373 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:49:31,375 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:49:31,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,375 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:31,376 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:49:31,376 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:49:31,376 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:49:31,377 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:49:31,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,377 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:31,377 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:49:31,378 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:49:31,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,379 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:31,379 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:49:31,380 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:49:31,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:31,380 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:31,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:49:31,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740
2024-11-20T06:49:31,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740
2024-11-20T06:49:31,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:49:31,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:49:31,383 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:49:31,384 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:49:31,384 DEBUG [RS:0;61c2066a8a0f:40669 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4653fb72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:49:31,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:49:31,386 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:49:31,387 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738424, jitterRate=-0.0610458105802536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:49:31,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085371372Initializing all the Stores at 1732085371373 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085371373Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085371373Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085371373Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085371373Cleaning up temporary data from old regions at 1732085371383 (+10 ms)Region opened successfully at 1732085371387 (+4 ms)
2024-11-20T06:49:31,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:49:31,388 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:49:31,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:49:31,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:49:31,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:49:31,396 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:40669
2024-11-20T06:49:31,396 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:49:31,396 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:49:31,396 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:49:31,396 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:49:31,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085371388Disabling compacts and flushes for region at 1732085371388Disabling writes for close at 1732085371388Writing region close event to WAL at 1732085371396 (+8 ms)Closed at 1732085371396
2024-11-20T06:49:31,397 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,38673,1732085370902 with port=40669, startcode=1732085371045
2024-11-20T06:49:31,397 DEBUG [RS:0;61c2066a8a0f:40669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:49:31,398 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:49:31,398 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:49:31,398 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:49:31,399 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:49:31,399 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54093, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:49:31,400 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,400 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,400 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:49:31,401 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942
2024-11-20T06:49:31,401 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45627
2024-11-20T06:49:31,401 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:49:31,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:49:31,408 DEBUG [RS:0;61c2066a8a0f:40669 {}] zookeeper.ZKUtil(111): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,409 WARN  [RS:0;61c2066a8a0f:40669 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:49:31,409 INFO  [RS:0;61c2066a8a0f:40669 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:49:31,409 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,409 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,40669,1732085371045]
2024-11-20T06:49:31,412 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:49:31,413 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:49:31,413 INFO  [RS:0;61c2066a8a0f:40669 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:49:31,414 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,414 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:49:31,415 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:49:31,415 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:49:31,415 DEBUG [RS:0;61c2066a8a0f:40669 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:49:31,420 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,421 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,421 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,421 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,421 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,421 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40669,1732085371045-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:49:31,436 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:49:31,436 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40669,1732085371045-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,436 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,437 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.Replication(171): 61c2066a8a0f,40669,1732085371045 started
2024-11-20T06:49:31,452 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:31,452 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,40669,1732085371045, RpcServer on 61c2066a8a0f/172.17.0.2:40669, sessionid=0x101584ae2c90001
2024-11-20T06:49:31,452 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:49:31,452 DEBUG [RS:0;61c2066a8a0f:40669 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,452 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,40669,1732085371045'
2024-11-20T06:49:31,452 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,40669,1732085371045'
2024-11-20T06:49:31,453 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:49:31,454 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:49:31,454 DEBUG [RS:0;61c2066a8a0f:40669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:49:31,454 INFO  [RS:0;61c2066a8a0f:40669 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:49:31,454 INFO  [RS:0;61c2066a8a0f:40669 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:49:31,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:31,551 WARN  [61c2066a8a0f:38673 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:49:31,557 INFO  [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C40669%2C1732085371045, suffix=, logDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045, archiveDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/oldWALs, maxLogs=32
2024-11-20T06:49:31,558 INFO  [RS:0;61c2066a8a0f:40669 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C40669%2C1732085371045.1732085371557
2024-11-20T06:49:31,567 INFO  [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045/61c2066a8a0f%2C40669%2C1732085371045.1732085371557
2024-11-20T06:49:31,568 DEBUG [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44703:44703),(127.0.0.1/127.0.0.1:40257:40257)]
2024-11-20T06:49:31,801 DEBUG [61c2066a8a0f:38673 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:49:31,801 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:31,803 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,40669,1732085371045, state=OPENING
2024-11-20T06:49:31,830 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:49:31,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:49:31,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:49:31,842 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:49:31,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:49:31,842 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,40669,1732085371045}]
2024-11-20T06:49:31,996 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:49:31,998 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53777, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:49:32,003 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:49:32,004 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:49:32,006 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C40669%2C1732085371045.meta, suffix=.meta, logDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045, archiveDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/oldWALs, maxLogs=32
2024-11-20T06:49:32,007 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C40669%2C1732085371045.meta.1732085372007.meta
2024-11-20T06:49:32,012 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045/61c2066a8a0f%2C40669%2C1732085371045.meta.1732085372007.meta
2024-11-20T06:49:32,013 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40257:40257),(127.0.0.1/127.0.0.1:44703:44703)]
2024-11-20T06:49:32,014 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:49:32,014 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:49:32,014 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:49:32,015 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:49:32,015 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:49:32,015 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:32,015 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:49:32,015 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:49:32,016 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:49:32,017 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:49:32,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,018 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:32,018 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:49:32,019 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:49:32,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,019 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:32,019 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:49:32,020 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:49:32,020 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,021 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:32,021 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:49:32,021 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:49:32,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,022 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:49:32,022 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:49:32,023 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740
2024-11-20T06:49:32,024 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740
2024-11-20T06:49:32,025 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:49:32,025 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:49:32,026 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:49:32,027 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:49:32,028 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842182, jitterRate=0.07089008390903473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:49:32,028 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:49:32,029 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085372015Writing region info on filesystem at 1732085372015Initializing all the Stores at 1732085372016 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085372016Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085372016Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085372016Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085372016Cleaning up temporary data from old regions at 1732085372025 (+9 ms)Running coprocessor post-open hooks at 1732085372028 (+3 ms)Region opened successfully at 1732085372028
2024-11-20T06:49:32,029 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085371995
2024-11-20T06:49:32,032 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:49:32,032 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:49:32,033 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:32,034 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,40669,1732085371045, state=OPEN
2024-11-20T06:49:32,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:49:32,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:49:32,096 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:32,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:49:32,096 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:49:32,099 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:49:32,099 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,40669,1732085371045 in 254 msec
2024-11-20T06:49:32,102 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:49:32,102 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 701 msec
2024-11-20T06:49:32,103 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:49:32,103 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:49:32,104 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:49:32,104 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,40669,1732085371045, seqNum=-1]
2024-11-20T06:49:32,104 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:49:32,106 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:49:32,111 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 765 msec
2024-11-20T06:49:32,111 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085372111, completionTime=-1
2024-11-20T06:49:32,111 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:49:32,111 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:49:32,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:49:32,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085432113
2024-11-20T06:49:32,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085492113
2024-11-20T06:49:32,113 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:38673, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,114 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,116 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.010sec
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:49:32,118 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:49:32,121 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:49:32,121 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:49:32,121 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,38673,1732085370902-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:49:32,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d663ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:49:32,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,38673,-1 for getting cluster id
2024-11-20T06:49:32,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:49:32,170 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8192399d-b065-432f-b184-365d049b3a13'
2024-11-20T06:49:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:49:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8192399d-b065-432f-b184-365d049b3a13"

2024-11-20T06:49:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d12f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:49:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,38673,-1]
2024-11-20T06:49:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:49:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:49:32,173 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:49:32,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651a5cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:49:32,174 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:49:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,40669,1732085371045, seqNum=-1]
2024-11-20T06:49:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:49:32,177 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:49:32,179 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:32,180 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:49:32,183 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:49:32,183 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-20T06:49:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,38673,1732085370902
2024-11-20T06:49:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57f750bd
2024-11-20T06:49:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-20T06:49:32,186 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-20T06:49:32,186 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-20T06:49:32,186 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-20T06:49:32,186 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:49:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-20T06:49:32,189 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-20T06:49:32,189 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,189 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-20T06:49:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:49:32,190 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-20T06:49:32,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741835_1011 (size=381)
2024-11-20T06:49:32,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741835_1011 (size=381)
2024-11-20T06:49:32,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:32,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:32,602 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a0951ded7cf07938c682cc129284f97, NAME => 'TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942
2024-11-20T06:49:32,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741836_1012 (size=64)
2024-11-20T06:49:32,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741836_1012 (size=64)
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 4a0951ded7cf07938c682cc129284f97, disabling compactions & flushes
2024-11-20T06:49:32,609 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. after waiting 0 ms
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,609 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,609 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a0951ded7cf07938c682cc129284f97:
Waiting for close lock at 1732085372609Disabling compacts and flushes for region at 1732085372609Disabling writes for close at 1732085372609Writing region close event to WAL at 1732085372609Closed at 1732085372609
2024-11-20T06:49:32,610 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-20T06:49:32,610 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732085372610"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085372610"}]},"ts":"1732085372610"}
2024-11-20T06:49:32,613 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-20T06:49:32,614 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-20T06:49:32,614 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085372614"}]},"ts":"1732085372614"}
2024-11-20T06:49:32,617 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-20T06:49:32,617 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, ASSIGN}]
2024-11-20T06:49:32,619 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, ASSIGN
2024-11-20T06:49:32,619 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, ASSIGN; state=OFFLINE, location=61c2066a8a0f,40669,1732085371045; forceNewPlan=false, retain=false
2024-11-20T06:49:32,770 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4a0951ded7cf07938c682cc129284f97, regionState=OPENING, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:32,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, ASSIGN because future has completed
2024-11-20T06:49:32,773 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045}]
2024-11-20T06:49:32,931 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,931 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a0951ded7cf07938c682cc129284f97, NAME => 'TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:49:32,932 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,932 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:32,932 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,932 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,934 INFO  [StoreOpener-4a0951ded7cf07938c682cc129284f97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4a0951ded7cf07938c682cc129284f97 
2024-11-20T06:49:32,936 INFO  [StoreOpener-4a0951ded7cf07938c682cc129284f97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a0951ded7cf07938c682cc129284f97 columnFamilyName info
2024-11-20T06:49:32,936 DEBUG [StoreOpener-4a0951ded7cf07938c682cc129284f97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:32,937 INFO  [StoreOpener-4a0951ded7cf07938c682cc129284f97-1 {}] regionserver.HStore(327): Store=4a0951ded7cf07938c682cc129284f97/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:32,937 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,938 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,939 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,939 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,939 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,942 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,945 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:49:32,945 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4a0951ded7cf07938c682cc129284f97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=770300, jitterRate=-0.020513057708740234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:49:32,945 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:32,946 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4a0951ded7cf07938c682cc129284f97:
Running coprocessor pre-open hook at 1732085372932Writing region info on filesystem at 1732085372932Initializing all the Stores at 1732085372933 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085372934 (+1 ms)Cleaning up temporary data from old regions at 1732085372939 (+5 ms)Running coprocessor post-open hooks at 1732085372945 (+6 ms)Region opened successfully at 1732085372946 (+1 ms)
2024-11-20T06:49:32,948 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., pid=6, masterSystemTime=1732085372926
2024-11-20T06:49:32,950 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,950 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:32,951 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4a0951ded7cf07938c682cc129284f97, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:32,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045 because future has completed
2024-11-20T06:49:32,958 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-20T06:49:32,958 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045 in 182 msec
2024-11-20T06:49:32,962 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-20T06:49:32,962 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, ASSIGN in 341 msec
2024-11-20T06:49:32,963 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-20T06:49:32,964 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085372964"}]},"ts":"1732085372964"}
2024-11-20T06:49:32,966 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-20T06:49:32,968 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-20T06:49:32,970 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 782 msec
2024-11-20T06:49:33,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-20T06:49:33,217 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-20T06:49:33,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-20T06:49:33,367 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:33,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:33,630 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,630 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,630 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,631 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,631 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,631 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,632 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,632 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,665 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,665 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,665 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:33,668 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,173 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:49:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,176 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,176 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,200 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,201 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,205 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,206 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,206 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:34,368 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:34,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:35,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:35,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:36,369 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:36,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:37,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:37,412 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:49:37,412 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-20T06:49:37,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:38,370 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:38,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:39,371 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:39,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:40,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:40,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:41,372 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:41,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38673 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-20T06:49:42,281 INFO  [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-20T06:49:42,281 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-20T06:49:42,284 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-20T06:49:42,284 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., hostname=61c2066a8a0f,40669,1732085371045, seqNum=2]
2024-11-20T06:49:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:42,304 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:49:42,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/226c32dfe7764a9294431a637ed46b28 is 1080, key is row0001/info:/1732085382289/Put/seqid=0
2024-11-20T06:49:42,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741837_1013 (size=12509)
2024-11-20T06:49:42,335 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741837_1013 (size=12509)
2024-11-20T06:49:42,336 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/226c32dfe7764a9294431a637ed46b28
2024-11-20T06:49:42,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/226c32dfe7764a9294431a637ed46b28 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28
2024-11-20T06:49:42,351 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28, entries=7, sequenceid=11, filesize=12.2 K
2024-11-20T06:49:42,352 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for 4a0951ded7cf07938c682cc129284f97 in 49ms, sequenceid=11, compaction requested=false
2024-11-20T06:49:42,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:42,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:42,354 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-20T06:49:42,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/b315d9049aea492a96d2721ecc4b84ae is 1080, key is row0008/info:/1732085382305/Put/seqid=0
2024-11-20T06:49:42,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741838_1014 (size=25453)
2024-11-20T06:49:42,371 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741838_1014 (size=25453)
2024-11-20T06:49:42,372 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/b315d9049aea492a96d2721ecc4b84ae
2024-11-20T06:49:42,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:42,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/b315d9049aea492a96d2721ecc4b84ae as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae
2024-11-20T06:49:42,386 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae, entries=19, sequenceid=33, filesize=24.9 K
2024-11-20T06:49:42,387 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for 4a0951ded7cf07938c682cc129284f97 in 33ms, sequenceid=33, compaction requested=false
2024-11-20T06:49:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=37.1 K, sizeToCheck=16.0 K
2024-11-20T06:49:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:42,387 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae because midkey is the same as first or last row
2024-11-20T06:49:42,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:43,373 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:43,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:44,372 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:49:44,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:44,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/2c83858580fc4d8aa776329b072db7f8 is 1080, key is row0027/info:/1732085382355/Put/seqid=0
2024-11-20T06:49:44,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741839_1015 (size=12509)
2024-11-20T06:49:44,382 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/2c83858580fc4d8aa776329b072db7f8
2024-11-20T06:49:44,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741839_1015 (size=12509)
2024-11-20T06:49:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/2c83858580fc4d8aa776329b072db7f8 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8
2024-11-20T06:49:44,397 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8, entries=7, sequenceid=43, filesize=12.2 K
2024-11-20T06:49:44,397 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 4a0951ded7cf07938c682cc129284f97 in 25ms, sequenceid=43, compaction requested=true
2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae because midkey is the same as first or last row
2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a0951ded7cf07938c682cc129284f97:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:49:44,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:44,398 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:44,399 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T06:49:44,400 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:49:44,400 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): 4a0951ded7cf07938c682cc129284f97/info is initiating minor compaction (all files)
2024-11-20T06:49:44,400 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a0951ded7cf07938c682cc129284f97/info in TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:44,401 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp, totalSize=49.3 K
2024-11-20T06:49:44,401 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226c32dfe7764a9294431a637ed46b28, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732085382289
2024-11-20T06:49:44,402 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b315d9049aea492a96d2721ecc4b84ae, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732085382305
2024-11-20T06:49:44,402 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c83858580fc4d8aa776329b072db7f8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732085382355
2024-11-20T06:49:44,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/3a8d1a9bc22c4aa09022fb59f023e3f3 is 1080, key is row0034/info:/1732085384373/Put/seqid=0
2024-11-20T06:49:44,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741840_1016 (size=16817)
2024-11-20T06:49:44,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741840_1016 (size=16817)
2024-11-20T06:49:44,425 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/3a8d1a9bc22c4aa09022fb59f023e3f3
2024-11-20T06:49:44,429 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a0951ded7cf07938c682cc129284f97#info#compaction#60 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:44,430 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/a3ada8dafc3547899db4b735c0d9b2db is 1080, key is row0001/info:/1732085382289/Put/seqid=0
2024-11-20T06:49:44,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/3a8d1a9bc22c4aa09022fb59f023e3f3 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3
2024-11-20T06:49:44,442 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3, entries=11, sequenceid=57, filesize=16.4 K
2024-11-20T06:49:44,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741841_1017 (size=40670)
2024-11-20T06:49:44,443 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=16.81 KB/17216 for 4a0951ded7cf07938c682cc129284f97 in 44ms, sequenceid=57, compaction requested=false
2024-11-20T06:49:44,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741841_1017 (size=40670)
2024-11-20T06:49:44,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:44,443 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.7 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,443 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,443 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae because midkey is the same as first or last row
2024-11-20T06:49:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:44,446 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-20T06:49:44,452 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/a3ada8dafc3547899db4b735c0d9b2db as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db
2024-11-20T06:49:44,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/4fac314fcfc5423d9a2a79c7ce621c32 is 1080, key is row0045/info:/1732085384401/Put/seqid=0
2024-11-20T06:49:44,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741842_1018 (size=23299)
2024-11-20T06:49:44,460 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741842_1018 (size=23299)
2024-11-20T06:49:44,461 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/4fac314fcfc5423d9a2a79c7ce621c32
2024-11-20T06:49:44,461 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a0951ded7cf07938c682cc129284f97/info of 4a0951ded7cf07938c682cc129284f97 into a3ada8dafc3547899db4b735c0d9b2db(size=39.7 K), total size for store is 56.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:44,461 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:44,461 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., storeName=4a0951ded7cf07938c682cc129284f97/info, priority=13, startTime=1732085384398; duration=0sec
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db because midkey is the same as first or last row
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db because midkey is the same as first or last row
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db because midkey is the same as first or last row
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:44,462 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a0951ded7cf07938c682cc129284f97:info
2024-11-20T06:49:44,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/4fac314fcfc5423d9a2a79c7ce621c32 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32
2024-11-20T06:49:44,473 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32, entries=17, sequenceid=77, filesize=22.8 K
2024-11-20T06:49:44,474 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=3.15 KB/3228 for 4a0951ded7cf07938c682cc129284f97 in 28ms, sequenceid=77, compaction requested=true
2024-11-20T06:49:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.9 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,474 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db because midkey is the same as first or last row
2024-11-20T06:49:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a0951ded7cf07938c682cc129284f97:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:49:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:44,475 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:44,476 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 80786 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:49:44,476 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): 4a0951ded7cf07938c682cc129284f97/info is initiating minor compaction (all files)
2024-11-20T06:49:44,476 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a0951ded7cf07938c682cc129284f97/info in TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:44,476 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp, totalSize=78.9 K
2024-11-20T06:49:44,476 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ada8dafc3547899db4b735c0d9b2db, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732085382289
2024-11-20T06:49:44,477 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a8d1a9bc22c4aa09022fb59f023e3f3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732085384373
2024-11-20T06:49:44,477 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fac314fcfc5423d9a2a79c7ce621c32, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732085384401
2024-11-20T06:49:44,490 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a0951ded7cf07938c682cc129284f97#info#compaction#62 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:44,490 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/05d2b9ed6d1e40778ce35c712d29d064 is 1080, key is row0001/info:/1732085382289/Put/seqid=0
2024-11-20T06:49:44,499 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741843_1019 (size=71001)
2024-11-20T06:49:44,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741843_1019 (size=71001)
2024-11-20T06:49:44,506 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/05d2b9ed6d1e40778ce35c712d29d064 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064
2024-11-20T06:49:44,512 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a0951ded7cf07938c682cc129284f97/info of 4a0951ded7cf07938c682cc129284f97 into 05d2b9ed6d1e40778ce35c712d29d064(size=69.3 K), total size for store is 69.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:44,512 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., storeName=4a0951ded7cf07938c682cc129284f97/info, priority=13, startTime=1732085384475; duration=0sec
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=69.3 K, sizeToCheck=16.0 K
2024-11-20T06:49:44,512 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:44,513 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:44,513 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:44,513 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a0951ded7cf07938c682cc129284f97:info
2024-11-20T06:49:44,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:45,374 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:45,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:46,375 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,463 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:49:46,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/7909db03bbbe478e8e521ec98325596f is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:49:46,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741844_1020 (size=12509)
2024-11-20T06:49:46,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741844_1020 (size=12509)
2024-11-20T06:49:46,473 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/7909db03bbbe478e8e521ec98325596f
2024-11-20T06:49:46,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/7909db03bbbe478e8e521ec98325596f as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f
2024-11-20T06:49:46,486 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f, entries=7, sequenceid=89, filesize=12.2 K
2024-11-20T06:49:46,487 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 4a0951ded7cf07938c682cc129284f97 in 24ms, sequenceid=89, compaction requested=false
2024-11-20T06:49:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=81.6 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,487 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,488 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T06:49:46,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/71f70f758fb94232a4bfea43d4435eac is 1080, key is row0069/info:/1732085386464/Put/seqid=0
2024-11-20T06:49:46,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741845_1021 (size=16817)
2024-11-20T06:49:46,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741845_1021 (size=16817)
2024-11-20T06:49:46,501 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/71f70f758fb94232a4bfea43d4435eac
2024-11-20T06:49:46,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/71f70f758fb94232a4bfea43d4435eac as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac
2024-11-20T06:49:46,513 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac, entries=11, sequenceid=103, filesize=16.4 K
2024-11-20T06:49:46,514 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for 4a0951ded7cf07938c682cc129284f97 in 27ms, sequenceid=103, compaction requested=true
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=98.0 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:46,514 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a0951ded7cf07938c682cc129284f97:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:49:46,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:46,515 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T06:49:46,516 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 100327 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:49:46,516 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): 4a0951ded7cf07938c682cc129284f97/info is initiating minor compaction (all files)
2024-11-20T06:49:46,516 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a0951ded7cf07938c682cc129284f97/info in TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:46,516 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp, totalSize=98.0 K
2024-11-20T06:49:46,516 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d2b9ed6d1e40778ce35c712d29d064, keycount=61, bloomtype=ROW, size=69.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732085382289
2024-11-20T06:49:46,517 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7909db03bbbe478e8e521ec98325596f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732085384448
2024-11-20T06:49:46,517 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71f70f758fb94232a4bfea43d4435eac, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732085386464
2024-11-20T06:49:46,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/ac6b0104d658482983f365c501d2231c is 1080, key is row0080/info:/1732085386489/Put/seqid=0
2024-11-20T06:49:46,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741846_1022 (size=17894)
2024-11-20T06:49:46,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741846_1022 (size=17894)
2024-11-20T06:49:46,529 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/ac6b0104d658482983f365c501d2231c
2024-11-20T06:49:46,532 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a0951ded7cf07938c682cc129284f97#info#compaction#66 average throughput is 27.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:46,533 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/375092727ba4409f9acbe0cdf65d0fac is 1080, key is row0001/info:/1732085382289/Put/seqid=0
2024-11-20T06:49:46,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/ac6b0104d658482983f365c501d2231c as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/ac6b0104d658482983f365c501d2231c
2024-11-20T06:49:46,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741847_1023 (size=90562)
2024-11-20T06:49:46,538 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741847_1023 (size=90562)
2024-11-20T06:49:46,542 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/ac6b0104d658482983f365c501d2231c, entries=12, sequenceid=118, filesize=17.5 K
2024-11-20T06:49:46,543 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for 4a0951ded7cf07938c682cc129284f97 in 28ms, sequenceid=118, compaction requested=false
2024-11-20T06:49:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=115.5 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 because midkey is the same as first or last row
2024-11-20T06:49:46,549 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/375092727ba4409f9acbe0cdf65d0fac as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac
2024-11-20T06:49:46,556 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a0951ded7cf07938c682cc129284f97/info of 4a0951ded7cf07938c682cc129284f97 into 375092727ba4409f9acbe0cdf65d0fac(size=88.4 K), total size for store is 105.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a0951ded7cf07938c682cc129284f97:

2024-11-20T06:49:46,556 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., storeName=4a0951ded7cf07938c682cc129284f97/info, priority=13, startTime=1732085386514; duration=0sec
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-20T06:49:46,556 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-20T06:49:46,557 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:46,557 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:46,557 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a0951ded7cf07938c682cc129284f97:info
2024-11-20T06:49:46,558 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] assignment.AssignmentManager(1363): Split request from 61c2066a8a0f,40669,1732085371045, parent={ENCODED => 4a0951ded7cf07938c682cc129284f97, NAME => 'TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-20T06:49:46,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:46,563 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:46,567 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4a0951ded7cf07938c682cc129284f97, daughterA=0c3c5b16ba3ecc9dc62bd758dd3a7764, daughterB=b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:46,568 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4a0951ded7cf07938c682cc129284f97, daughterA=0c3c5b16ba3ecc9dc62bd758dd3a7764, daughterB=b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:46,568 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4a0951ded7cf07938c682cc129284f97, daughterA=0c3c5b16ba3ecc9dc62bd758dd3a7764, daughterB=b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:46,568 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4a0951ded7cf07938c682cc129284f97, daughterA=0c3c5b16ba3ecc9dc62bd758dd3a7764, daughterB=b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:46,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, UNASSIGN}]
2024-11-20T06:49:46,577 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, UNASSIGN
2024-11-20T06:49:46,578 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4a0951ded7cf07938c682cc129284f97, regionState=CLOSING, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:46,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, UNASSIGN because future has completed
2024-11-20T06:49:46,581 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-20T06:49:46,581 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045}]
2024-11-20T06:49:46,739 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,739 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-20T06:49:46,740 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 4a0951ded7cf07938c682cc129284f97, disabling compactions & flushes
2024-11-20T06:49:46,740 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:46,740 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:46,740 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. after waiting 0 ms
2024-11-20T06:49:46,740 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:46,740 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 4a0951ded7cf07938c682cc129284f97 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB
2024-11-20T06:49:46,746 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/e1f611fb594a43488e8ce8a3015c0aa4 is 1080, key is row0092/info:/1732085386516/Put/seqid=0
2024-11-20T06:49:46,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741848_1024 (size=10347)
2024-11-20T06:49:46,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741848_1024 (size=10347)
2024-11-20T06:49:46,753 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/e1f611fb594a43488e8ce8a3015c0aa4
2024-11-20T06:49:46,759 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/.tmp/info/e1f611fb594a43488e8ce8a3015c0aa4 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/e1f611fb594a43488e8ce8a3015c0aa4
2024-11-20T06:49:46,765 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/e1f611fb594a43488e8ce8a3015c0aa4, entries=5, sequenceid=127, filesize=10.1 K
2024-11-20T06:49:46,766 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 4a0951ded7cf07938c682cc129284f97 in 26ms, sequenceid=127, compaction requested=true
2024-11-20T06:49:46,768 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac] to archive
2024-11-20T06:49:46,769 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:49:46,771 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/226c32dfe7764a9294431a637ed46b28
2024-11-20T06:49:46,772 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/b315d9049aea492a96d2721ecc4b84ae
2024-11-20T06:49:46,774 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/a3ada8dafc3547899db4b735c0d9b2db
2024-11-20T06:49:46,775 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/2c83858580fc4d8aa776329b072db7f8
2024-11-20T06:49:46,777 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/3a8d1a9bc22c4aa09022fb59f023e3f3
2024-11-20T06:49:46,778 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/05d2b9ed6d1e40778ce35c712d29d064
2024-11-20T06:49:46,779 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/4fac314fcfc5423d9a2a79c7ce621c32
2024-11-20T06:49:46,781 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/7909db03bbbe478e8e521ec98325596f
2024-11-20T06:49:46,782 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/71f70f758fb94232a4bfea43d4435eac
2024-11-20T06:49:46,789 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-20T06:49:46,790 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.
2024-11-20T06:49:46,790 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 4a0951ded7cf07938c682cc129284f97:
Waiting for close lock at 1732085386740Running coprocessor pre-close hooks at 1732085386740Disabling compacts and flushes for region at 1732085386740Disabling writes for close at 1732085386740Obtaining lock to block concurrent updates at 1732085386740Preparing flush snapshotting stores in 4a0951ded7cf07938c682cc129284f97 at 1732085386740Finished memstore snapshotting TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., syncing WAL and waiting on mvcc, flushsize=dataSize=5380, getHeapSize=6000, getOffHeapSize=0, getCellsCount=5 at 1732085386741 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. at 1732085386742 (+1 ms)Flushing 4a0951ded7cf07938c682cc129284f97/info: creating writer at 1732085386742Flushing 4a0951ded7cf07938c682cc129284f97/info: appending metadata at 1732085386746 (+4 ms)Flushing 4a0951ded7cf07938c682cc129284f97/info: closing flushed file at 1732085386746Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a171d5: reopening flushed file at 1732085386758 (+12 ms)Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 4a0951ded7cf07938c682cc129284f97 in 26ms, sequenceid=127, compaction requested=true at 1732085386766 (+8 ms)Writing region close event to WAL at 1732085386785 (+19 ms)Running coprocessor post-close hooks at 1732085386790 (+5 ms)Closed at 1732085386790
2024-11-20T06:49:46,794 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,795 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=4a0951ded7cf07938c682cc129284f97, regionState=CLOSED
2024-11-20T06:49:46,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045 because future has completed
2024-11-20T06:49:46,800 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-20T06:49:46,800 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a0951ded7cf07938c682cc129284f97, server=61c2066a8a0f,40669,1732085371045 in 217 msec
2024-11-20T06:49:46,802 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-20T06:49:46,802 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4a0951ded7cf07938c682cc129284f97, UNASSIGN in 225 msec
2024-11-20T06:49:46,810 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:46,813 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=4a0951ded7cf07938c682cc129284f97, threads=3
2024-11-20T06:49:46,816 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/e1f611fb594a43488e8ce8a3015c0aa4 for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,816 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,816 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/ac6b0104d658482983f365c501d2231c for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,826 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/e1f611fb594a43488e8ce8a3015c0aa4, top=true
2024-11-20T06:49:46,828 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/ac6b0104d658482983f365c501d2231c, top=true
2024-11-20T06:49:46,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741849_1025 (size=27)
2024-11-20T06:49:46,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741849_1025 (size=27)
2024-11-20T06:49:46,832 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4 for child: b6811eaddd93e7ae02da7b00eebbc02f, parent: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,832 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/e1f611fb594a43488e8ce8a3015c0aa4 for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,833 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c for child: b6811eaddd93e7ae02da7b00eebbc02f, parent: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,833 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/ac6b0104d658482983f365c501d2231c for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741850_1026 (size=27)
2024-11-20T06:49:46,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741850_1026 (size=27)
2024-11-20T06:49:46,841 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac for region: 4a0951ded7cf07938c682cc129284f97
2024-11-20T06:49:46,843 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 4a0951ded7cf07938c682cc129284f97 Daughter A: [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97] storefiles, Daughter B: [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4] storefiles.
2024-11-20T06:49:46,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741851_1027 (size=71)
2024-11-20T06:49:46,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741851_1027 (size=71)
2024-11-20T06:49:46,853 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:46,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741852_1028 (size=71)
2024-11-20T06:49:46,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741852_1028 (size=71)
2024-11-20T06:49:46,867 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:46,877 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-20T06:49:46,880 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-20T06:49:46,883 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732085386882"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732085386882"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732085386882"}]},"ts":"1732085386882"}
2024-11-20T06:49:46,883 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732085386882"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085386882"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732085386882"}]},"ts":"1732085386882"}
2024-11-20T06:49:46,883 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732085386882"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085386882"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732085386882"}]},"ts":"1732085386882"}
2024-11-20T06:49:46,902 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0c3c5b16ba3ecc9dc62bd758dd3a7764, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f, ASSIGN}]
2024-11-20T06:49:46,904 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f, ASSIGN
2024-11-20T06:49:46,904 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0c3c5b16ba3ecc9dc62bd758dd3a7764, ASSIGN
2024-11-20T06:49:46,905 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f, ASSIGN; state=SPLITTING_NEW, location=61c2066a8a0f,40669,1732085371045; forceNewPlan=false, retain=false
2024-11-20T06:49:46,905 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0c3c5b16ba3ecc9dc62bd758dd3a7764, ASSIGN; state=SPLITTING_NEW, location=61c2066a8a0f,40669,1732085371045; forceNewPlan=false, retain=false
2024-11-20T06:49:47,055 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b6811eaddd93e7ae02da7b00eebbc02f, regionState=OPENING, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:47,055 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0c3c5b16ba3ecc9dc62bd758dd3a7764, regionState=OPENING, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:47,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0c3c5b16ba3ecc9dc62bd758dd3a7764, ASSIGN because future has completed
2024-11-20T06:49:47,058 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c3c5b16ba3ecc9dc62bd758dd3a7764, server=61c2066a8a0f,40669,1732085371045}]
2024-11-20T06:49:47,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f, ASSIGN because future has completed
2024-11-20T06:49:47,102 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045}]
2024-11-20T06:49:47,258 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:47,258 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => b6811eaddd93e7ae02da7b00eebbc02f, NAME => 'TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-20T06:49:47,259 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,259 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:47,259 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,259 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,261 INFO  [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b6811eaddd93e7ae02da7b00eebbc02f 
2024-11-20T06:49:47,262 INFO  [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6811eaddd93e7ae02da7b00eebbc02f columnFamilyName info
2024-11-20T06:49:47,262 DEBUG [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:47,278 DEBUG [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-top
2024-11-20T06:49:47,283 DEBUG [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c
2024-11-20T06:49:47,287 DEBUG [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4
2024-11-20T06:49:47,287 INFO  [StoreOpener-b6811eaddd93e7ae02da7b00eebbc02f-1 {}] regionserver.HStore(327): Store=b6811eaddd93e7ae02da7b00eebbc02f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:47,287 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,288 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,289 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,290 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,290 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,292 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,292 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened b6811eaddd93e7ae02da7b00eebbc02f; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717366, jitterRate=-0.0878220945596695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:49:47,293 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:47,293 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for b6811eaddd93e7ae02da7b00eebbc02f:
Running coprocessor pre-open hook at 1732085387259Writing region info on filesystem at 1732085387259Initializing all the Stores at 1732085387260 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085387260Cleaning up temporary data from old regions at 1732085387290 (+30 ms)Running coprocessor post-open hooks at 1732085387293 (+3 ms)Region opened successfully at 1732085387293
2024-11-20T06:49:47,294 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., pid=13, masterSystemTime=1732085387254
2024-11-20T06:49:47,294 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:49:47,294 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:47,294 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:47,296 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:47,296 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:49:47,296 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:47,296 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-top, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=116.0 K
2024-11-20T06:49:47,296 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:47,296 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:47,296 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:49:47,297 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 0c3c5b16ba3ecc9dc62bd758dd3a7764, NAME => 'TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-20T06:49:47,297 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732085382289
2024-11-20T06:49:47,297 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,297 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:49:47,297 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,297 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,297 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=b6811eaddd93e7ae02da7b00eebbc02f, regionState=OPEN, openSeqNum=131, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:47,297 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732085386489
2024-11-20T06:49:47,298 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732085386516
2024-11-20T06:49:47,298 INFO  [StoreOpener-0c3c5b16ba3ecc9dc62bd758dd3a7764-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0c3c5b16ba3ecc9dc62bd758dd3a7764 
2024-11-20T06:49:47,299 INFO  [StoreOpener-0c3c5b16ba3ecc9dc62bd758dd3a7764-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c3c5b16ba3ecc9dc62bd758dd3a7764 columnFamilyName info
2024-11-20T06:49:47,299 DEBUG [StoreOpener-0c3c5b16ba3ecc9dc62bd758dd3a7764-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:49:47,300 WARN  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38673 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=61c2066a8a0f,40669,1732085371045, table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now.
2024-11-20T06:49:47,300 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-20T06:49:47,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-20T06:49:47,301 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-20T06:49:47,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045 because future has completed
2024-11-20T06:49:47,307 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-20T06:49:47,307 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045 in 200 msec
2024-11-20T06:49:47,308 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=b6811eaddd93e7ae02da7b00eebbc02f, ASSIGN in 405 msec
2024-11-20T06:49:47,316 DEBUG [StoreOpener-0c3c5b16ba3ecc9dc62bd758dd3a7764-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-bottom
2024-11-20T06:49:47,316 INFO  [StoreOpener-0c3c5b16ba3ecc9dc62bd758dd3a7764-1 {}] regionserver.HStore(327): Store=0c3c5b16ba3ecc9dc62bd758dd3a7764/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:49:47,317 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,317 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,318 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,319 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,319 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/bd4f20680c12408cafd4fed4c8706a14 is 193, key is TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f./info:regioninfo/1732085387297/Put/seqid=0
2024-11-20T06:49:47,321 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,322 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 0c3c5b16ba3ecc9dc62bd758dd3a7764; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=747891, jitterRate=-0.04900863766670227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-20T06:49:47,322 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:49:47,322 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 0c3c5b16ba3ecc9dc62bd758dd3a7764:
Running coprocessor pre-open hook at 1732085387297Writing region info on filesystem at 1732085387297Initializing all the Stores at 1732085387298 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085387298Cleaning up temporary data from old regions at 1732085387319 (+21 ms)Running coprocessor post-open hooks at 1732085387322 (+3 ms)Region opened successfully at 1732085387322
2024-11-20T06:49:47,323 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764., pid=12, masterSystemTime=1732085387254
2024-11-20T06:49:47,323 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 0c3c5b16ba3ecc9dc62bd758dd3a7764:info, priority=-2147483648, current under compaction store size is 2
2024-11-20T06:49:47,323 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:47,323 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-20T06:49:47,324 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:49:47,324 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HStore(1541): 0c3c5b16ba3ecc9dc62bd758dd3a7764/info is initiating minor compaction (all files)
2024-11-20T06:49:47,324 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0c3c5b16ba3ecc9dc62bd758dd3a7764/info in TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:49:47,325 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-bottom] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/.tmp, totalSize=88.4 K
2024-11-20T06:49:47,325 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] compactions.Compactor(225): Compacting 375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97, keycount=39, bloomtype=ROW, size=88.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732085382289
2024-11-20T06:49:47,325 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:49:47,326 INFO  [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:49:47,326 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=0c3c5b16ba3ecc9dc62bd758dd3a7764, regionState=OPEN, openSeqNum=131, regionLocation=61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:47,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741853_1029 (size=9882)
2024-11-20T06:49:47,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741853_1029 (size=9882)
2024-11-20T06:49:47,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0c3c5b16ba3ecc9dc62bd758dd3a7764, server=61c2066a8a0f,40669,1732085371045 because future has completed
2024-11-20T06:49:47,330 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/bd4f20680c12408cafd4fed4c8706a14
2024-11-20T06:49:47,333 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#69 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:47,333 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/e5dd2998a97841d48f9e0cc9e5d5c729 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:49:47,334 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-20T06:49:47,334 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 0c3c5b16ba3ecc9dc62bd758dd3a7764, server=61c2066a8a0f,40669,1732085371045 in 274 msec
2024-11-20T06:49:47,336 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-20T06:49:47,336 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0c3c5b16ba3ecc9dc62bd758dd3a7764, ASSIGN in 432 msec
2024-11-20T06:49:47,338 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=4a0951ded7cf07938c682cc129284f97, daughterA=0c3c5b16ba3ecc9dc62bd758dd3a7764, daughterB=b6811eaddd93e7ae02da7b00eebbc02f in 773 msec
2024-11-20T06:49:47,347 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0c3c5b16ba3ecc9dc62bd758dd3a7764#info#compaction#70 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:47,347 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/.tmp/info/47d98c7a778d45619ae86779aa1014bf is 1080, key is row0001/info:/1732085382289/Put/seqid=0
2024-11-20T06:49:47,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741854_1030 (size=42984)
2024-11-20T06:49:47,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741854_1030 (size=42984)
2024-11-20T06:49:47,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741855_1031 (size=70862)
2024-11-20T06:49:47,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741855_1031 (size=70862)
2024-11-20T06:49:47,364 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/.tmp/info/47d98c7a778d45619ae86779aa1014bf as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/47d98c7a778d45619ae86779aa1014bf
2024-11-20T06:49:47,364 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/e5dd2998a97841d48f9e0cc9e5d5c729 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/e5dd2998a97841d48f9e0cc9e5d5c729
2024-11-20T06:49:47,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/ns/20c48b38ff5841d094a3e2ed0c07ab56 is 43, key is default/ns:d/1732085372106/Put/seqid=0
2024-11-20T06:49:47,371 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into e5dd2998a97841d48f9e0cc9e5d5c729(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:47,371 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 0c3c5b16ba3ecc9dc62bd758dd3a7764/info of 0c3c5b16ba3ecc9dc62bd758dd3a7764 into 47d98c7a778d45619ae86779aa1014bf(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:47,371 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:49:47,371 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0c3c5b16ba3ecc9dc62bd758dd3a7764:

2024-11-20T06:49:47,372 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085387294; duration=0sec
2024-11-20T06:49:47,372 INFO  [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764., storeName=0c3c5b16ba3ecc9dc62bd758dd3a7764/info, priority=15, startTime=1732085387323; duration=0sec
2024-11-20T06:49:47,372 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:47,372 DEBUG [RS:0;61c2066a8a0f:40669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0c3c5b16ba3ecc9dc62bd758dd3a7764:info
2024-11-20T06:49:47,372 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:47,372 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:49:47,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741856_1032 (size=5153)
2024-11-20T06:49:47,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741856_1032 (size=5153)
2024-11-20T06:49:47,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:47,377 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/ns/20c48b38ff5841d094a3e2ed0c07ab56
2024-11-20T06:49:47,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/table/458413311264429cad225041df9d174a is 65, key is TestLogRolling-testLogRolling/table:state/1732085372964/Put/seqid=0
2024-11-20T06:49:47,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741857_1033 (size=5340)
2024-11-20T06:49:47,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741857_1033 (size=5340)
2024-11-20T06:49:47,404 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/table/458413311264429cad225041df9d174a
2024-11-20T06:49:47,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/bd4f20680c12408cafd4fed4c8706a14 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/info/bd4f20680c12408cafd4fed4c8706a14
2024-11-20T06:49:47,415 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/info/bd4f20680c12408cafd4fed4c8706a14, entries=30, sequenceid=17, filesize=9.7 K
2024-11-20T06:49:47,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/ns/20c48b38ff5841d094a3e2ed0c07ab56 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/ns/20c48b38ff5841d094a3e2ed0c07ab56
2024-11-20T06:49:47,422 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/ns/20c48b38ff5841d094a3e2ed0c07ab56, entries=2, sequenceid=17, filesize=5.0 K
2024-11-20T06:49:47,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/table/458413311264429cad225041df9d174a as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/table/458413311264429cad225041df9d174a
2024-11-20T06:49:47,429 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/table/458413311264429cad225041df9d174a, entries=2, sequenceid=17, filesize=5.2 K
2024-11-20T06:49:47,430 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 129ms, sequenceid=17, compaction requested=false
2024-11-20T06:49:47,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-20T06:49:47,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:48,376 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:53268 deadline: 1732085398528, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. is not online on 61c2066a8a0f,40669,1732085371045
2024-11-20T06:49:48,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., hostname=61c2066a8a0f,40669,1732085371045, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., hostname=61c2066a8a0f,40669,1732085371045, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. is not online on 61c2066a8a0f,40669,1732085371045
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T06:49:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., hostname=61c2066a8a0f,40669,1732085371045, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97. is not online on 61c2066a8a0f,40669,1732085371045
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T06:49:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732085372186.4a0951ded7cf07938c682cc129284f97., hostname=61c2066a8a0f,40669,1732085371045, seqNum=2 from cache
2024-11-20T06:49:48,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:49,377 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:49,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:50,378 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:50,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:51,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:51,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:51,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,792 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,793 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,821 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,821 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:51,824 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,331 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-20T06:49:52,332 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,332 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,332 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,333 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,334 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,361 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-20T06:49:52,379 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:52,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:53,380 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:53,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:54,380 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:54,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:55,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:55,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:56,381 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:56,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:57,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:57,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:58,382 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:58,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., hostname=61c2066a8a0f,40669,1732085371045, seqNum=131]
2024-11-20T06:49:58,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:58,570 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:49:58,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e9dbdf908a45ca9f417e839de663a4 is 1080, key is row0097/info:/1732085398557/Put/seqid=0
2024-11-20T06:49:58,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741858_1034 (size=12516)
2024-11-20T06:49:58,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741858_1034 (size=12516)
2024-11-20T06:49:58,587 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e9dbdf908a45ca9f417e839de663a4
2024-11-20T06:49:58,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e9dbdf908a45ca9f417e839de663a4 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4
2024-11-20T06:49:58,606 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4, entries=7, sequenceid=141, filesize=12.2 K
2024-11-20T06:49:58,607 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for b6811eaddd93e7ae02da7b00eebbc02f in 37ms, sequenceid=141, compaction requested=false
2024-11-20T06:49:58,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:49:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:49:58,610 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-20T06:49:58,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1cef473351b14b518162bcc72ee07c1d is 1080, key is row0104/info:/1732085398572/Put/seqid=0
2024-11-20T06:49:58,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741859_1035 (size=22238)
2024-11-20T06:49:58,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741859_1035 (size=22238)
2024-11-20T06:49:58,621 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1cef473351b14b518162bcc72ee07c1d
2024-11-20T06:49:58,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1cef473351b14b518162bcc72ee07c1d as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d
2024-11-20T06:49:58,638 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d, entries=16, sequenceid=160, filesize=21.7 K
2024-11-20T06:49:58,639 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for b6811eaddd93e7ae02da7b00eebbc02f in 30ms, sequenceid=160, compaction requested=true
2024-11-20T06:49:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:49:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:49:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:58,639 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:49:58,640 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 77738 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:49:58,640 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:49:58,640 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:49:58,641 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/e5dd2998a97841d48f9e0cc9e5d5c729, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=75.9 K
2024-11-20T06:49:58,641 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5dd2998a97841d48f9e0cc9e5d5c729, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732085384448
2024-11-20T06:49:58,641 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e9dbdf908a45ca9f417e839de663a4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732085398557
2024-11-20T06:49:58,641 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cef473351b14b518162bcc72ee07c1d, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732085398572
2024-11-20T06:49:58,656 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#75 average throughput is 29.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:49:58,656 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/8ff4dac7b63e4ff0aab8bf622874e9ac is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:49:58,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741860_1036 (size=67948)
2024-11-20T06:49:58,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741860_1036 (size=67948)
2024-11-20T06:49:58,675 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/8ff4dac7b63e4ff0aab8bf622874e9ac as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8ff4dac7b63e4ff0aab8bf622874e9ac
2024-11-20T06:49:58,683 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 8ff4dac7b63e4ff0aab8bf622874e9ac(size=66.4 K), total size for store is 66.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:49:58,683 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:49:58,683 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085398639; duration=0sec
2024-11-20T06:49:58,683 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:49:58,683 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:49:59,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:49:59,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:00,383 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:00,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:00,633 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-20T06:50:00,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ffc78c591cf744509140f1d0c7f9415f is 1080, key is row0120/info:/1732085398612/Put/seqid=0
2024-11-20T06:50:00,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741861_1037 (size=15750)
2024-11-20T06:50:00,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741861_1037 (size=15750)
2024-11-20T06:50:00,646 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ffc78c591cf744509140f1d0c7f9415f
2024-11-20T06:50:00,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ffc78c591cf744509140f1d0c7f9415f as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f
2024-11-20T06:50:00,657 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f, entries=10, sequenceid=174, filesize=15.4 K
2024-11-20T06:50:00,658 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=7.36 KB/7532 for b6811eaddd93e7ae02da7b00eebbc02f in 25ms, sequenceid=174, compaction requested=false
2024-11-20T06:50:00,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:00,660 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-20T06:50:00,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/8005e9dd5c4f49c5ab7dff7723ba9540 is 1080, key is row0130/info:/1732085400634/Put/seqid=0
2024-11-20T06:50:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741862_1038 (size=14672)
2024-11-20T06:50:00,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741862_1038 (size=14672)
2024-11-20T06:50:00,669 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/8005e9dd5c4f49c5ab7dff7723ba9540
2024-11-20T06:50:00,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/8005e9dd5c4f49c5ab7dff7723ba9540 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540
2024-11-20T06:50:00,681 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540, entries=9, sequenceid=186, filesize=14.3 K
2024-11-20T06:50:00,681 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=9.46 KB/9684 for b6811eaddd93e7ae02da7b00eebbc02f in 21ms, sequenceid=186, compaction requested=true
2024-11-20T06:50:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:00,682 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:00,683 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 98370 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:00,683 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:00,683 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:00,683 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-20T06:50:00,683 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8ff4dac7b63e4ff0aab8bf622874e9ac, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=96.1 K
2024-11-20T06:50:00,684 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff4dac7b63e4ff0aab8bf622874e9ac, keycount=58, bloomtype=ROW, size=66.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732085384448
2024-11-20T06:50:00,684 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc78c591cf744509140f1d0c7f9415f, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732085398612
2024-11-20T06:50:00,684 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8005e9dd5c4f49c5ab7dff7723ba9540, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732085400634
2024-11-20T06:50:00,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ecefddc745634deaaafd5ea0be3b29ee is 1080, key is row0139/info:/1732085400661/Put/seqid=0
2024-11-20T06:50:00,705 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#79 average throughput is 39.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:00,706 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/25a82300ffee4e029459fc6f8d991012 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:00,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741863_1039 (size=15750)
2024-11-20T06:50:00,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741863_1039 (size=15750)
2024-11-20T06:50:00,714 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ecefddc745634deaaafd5ea0be3b29ee
2024-11-20T06:50:00,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/ecefddc745634deaaafd5ea0be3b29ee as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee
2024-11-20T06:50:00,727 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee, entries=10, sequenceid=199, filesize=15.4 K
2024-11-20T06:50:00,728 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=12.61 KB/12912 for b6811eaddd93e7ae02da7b00eebbc02f in 45ms, sequenceid=199, compaction requested=false
2024-11-20T06:50:00,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:00,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741864_1040 (size=88609)
2024-11-20T06:50:00,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741864_1040 (size=88609)
2024-11-20T06:50:00,754 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/25a82300ffee4e029459fc6f8d991012 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/25a82300ffee4e029459fc6f8d991012
2024-11-20T06:50:00,760 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 25a82300ffee4e029459fc6f8d991012(size=86.5 K), total size for store is 101.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:00,760 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:00,760 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085400681; duration=0sec
2024-11-20T06:50:00,760 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:00,760 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:00,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-20T06:50:01,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:01,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:02,384 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:02,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:02,721 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-20T06:50:02,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/c42e7575f1db452ab7e30fa883faba42 is 1080, key is row0149/info:/1732085400684/Put/seqid=0
2024-11-20T06:50:02,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741865_1041 (size=19000)
2024-11-20T06:50:02,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741865_1041 (size=19000)
2024-11-20T06:50:02,731 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/c42e7575f1db452ab7e30fa883faba42
2024-11-20T06:50:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/c42e7575f1db452ab7e30fa883faba42 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42
2024-11-20T06:50:02,743 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42, entries=13, sequenceid=216, filesize=18.6 K
2024-11-20T06:50:02,743 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for b6811eaddd93e7ae02da7b00eebbc02f in 22ms, sequenceid=216, compaction requested=true
2024-11-20T06:50:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:02,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:02,744 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:02,745 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T06:50:02,745 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123359 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:02,745 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:02,745 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:02,745 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/25a82300ffee4e029459fc6f8d991012, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=120.5 K
2024-11-20T06:50:02,746 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a82300ffee4e029459fc6f8d991012, keycount=77, bloomtype=ROW, size=86.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732085384448
2024-11-20T06:50:02,746 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecefddc745634deaaafd5ea0be3b29ee, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732085400661
2024-11-20T06:50:02,747 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42e7575f1db452ab7e30fa883faba42, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732085400684
2024-11-20T06:50:02,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e217e508214694803e1c5adb9ff48f is 1080, key is row0162/info:/1732085402722/Put/seqid=0
2024-11-20T06:50:02,761 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#82 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:02,762 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/21e4b137015a41fe93c7fda9cbf55d13 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:02,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741866_1042 (size=16828)
2024-11-20T06:50:02,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741866_1042 (size=16828)
2024-11-20T06:50:02,770 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e217e508214694803e1c5adb9ff48f
2024-11-20T06:50:02,771 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741867_1043 (size=113509)
2024-11-20T06:50:02,774 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741867_1043 (size=113509)
2024-11-20T06:50:02,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/f5e217e508214694803e1c5adb9ff48f as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f
2024-11-20T06:50:02,781 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/21e4b137015a41fe93c7fda9cbf55d13 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/21e4b137015a41fe93c7fda9cbf55d13
2024-11-20T06:50:02,785 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f, entries=11, sequenceid=230, filesize=16.4 K
2024-11-20T06:50:02,787 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=16.81 KB/17216 for b6811eaddd93e7ae02da7b00eebbc02f in 41ms, sequenceid=230, compaction requested=false
2024-11-20T06:50:02,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:02,788 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-20T06:50:02,789 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 21e4b137015a41fe93c7fda9cbf55d13(size=110.8 K), total size for store is 127.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:02,789 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:02,789 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085402744; duration=0sec
2024-11-20T06:50:02,789 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:02,789 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:02,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/7e88bf5798e44a0dae96273ce0b2ecb0 is 1080, key is row0173/info:/1732085402746/Put/seqid=0
2024-11-20T06:50:02,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741868_1044 (size=23316)
2024-11-20T06:50:02,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741868_1044 (size=23316)
2024-11-20T06:50:02,802 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/7e88bf5798e44a0dae96273ce0b2ecb0
2024-11-20T06:50:02,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/7e88bf5798e44a0dae96273ce0b2ecb0 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0
2024-11-20T06:50:02,815 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0, entries=17, sequenceid=250, filesize=22.8 K
2024-11-20T06:50:02,816 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=3.15 KB/3228 for b6811eaddd93e7ae02da7b00eebbc02f in 29ms, sequenceid=250, compaction requested=true
2024-11-20T06:50:02,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:02,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:02,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:02,816 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:02,817 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 153653 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:02,818 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:02,818 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:02,818 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/21e4b137015a41fe93c7fda9cbf55d13, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=150.1 K
2024-11-20T06:50:02,818 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e4b137015a41fe93c7fda9cbf55d13, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732085384448
2024-11-20T06:50:02,819 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5e217e508214694803e1c5adb9ff48f, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732085402722
2024-11-20T06:50:02,819 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e88bf5798e44a0dae96273ce0b2ecb0, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732085402746
2024-11-20T06:50:02,846 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#84 average throughput is 8.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:02,847 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/a45a020f40bd40ab9113a1b083005374 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:02,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741869_1045 (size=144000)
2024-11-20T06:50:02,861 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741869_1045 (size=144000)
2024-11-20T06:50:02,870 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/a45a020f40bd40ab9113a1b083005374 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/a45a020f40bd40ab9113a1b083005374
2024-11-20T06:50:02,877 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into a45a020f40bd40ab9113a1b083005374(size=140.6 K), total size for store is 140.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:02,877 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:02,877 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085402816; duration=0sec
2024-11-20T06:50:02,877 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:02,877 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:03,385 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:03,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:04,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:04,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:04,804 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:50:04,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/0bc38befba24455e8823fbc7fb297c9d is 1080, key is row0190/info:/1732085402789/Put/seqid=0
2024-11-20T06:50:04,822 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741870_1046 (size=12520)
2024-11-20T06:50:04,822 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741870_1046 (size=12520)
2024-11-20T06:50:04,822 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/0bc38befba24455e8823fbc7fb297c9d
2024-11-20T06:50:04,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/0bc38befba24455e8823fbc7fb297c9d as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d
2024-11-20T06:50:04,835 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d, entries=7, sequenceid=262, filesize=12.2 K
2024-11-20T06:50:04,836 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9684 for b6811eaddd93e7ae02da7b00eebbc02f in 32ms, sequenceid=262, compaction requested=false
2024-11-20T06:50:04,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:04,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:04,838 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-20T06:50:04,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1b1c217bc62f4709a27c7949b6d858e5 is 1080, key is row0197/info:/1732085404805/Put/seqid=0
2024-11-20T06:50:04,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741871_1047 (size=16839)
2024-11-20T06:50:04,849 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741871_1047 (size=16839)
2024-11-20T06:50:04,849 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1b1c217bc62f4709a27c7949b6d858e5
2024-11-20T06:50:04,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/1b1c217bc62f4709a27c7949b6d858e5 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5
2024-11-20T06:50:04,863 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5, entries=11, sequenceid=276, filesize=16.4 K
2024-11-20T06:50:04,864 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=11.56 KB/11836 for b6811eaddd93e7ae02da7b00eebbc02f in 26ms, sequenceid=276, compaction requested=true
2024-11-20T06:50:04,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:04,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:04,864 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:04,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:04,865 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T06:50:04,866 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 173359 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:04,866 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:04,866 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:04,866 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/a45a020f40bd40ab9113a1b083005374, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=169.3 K
2024-11-20T06:50:04,867 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a45a020f40bd40ab9113a1b083005374, keycount=128, bloomtype=ROW, size=140.6 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732085384448
2024-11-20T06:50:04,867 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc38befba24455e8823fbc7fb297c9d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732085402789
2024-11-20T06:50:04,867 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b1c217bc62f4709a27c7949b6d858e5, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732085404805
2024-11-20T06:50:04,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/80153bd122d74ef1ab726189dfd21208 is 1080, key is row0208/info:/1732085404839/Put/seqid=0
2024-11-20T06:50:04,876 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741872_1048 (size=17918)
2024-11-20T06:50:04,877 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741872_1048 (size=17918)
2024-11-20T06:50:04,877 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/80153bd122d74ef1ab726189dfd21208
2024-11-20T06:50:04,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/80153bd122d74ef1ab726189dfd21208 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208
2024-11-20T06:50:04,882 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#88 average throughput is 49.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:04,883 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/65218d679da9421c9941060467392698 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:04,887 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208, entries=12, sequenceid=291, filesize=17.5 K
2024-11-20T06:50:04,889 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for b6811eaddd93e7ae02da7b00eebbc02f in 23ms, sequenceid=291, compaction requested=false
2024-11-20T06:50:04,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:04,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741873_1049 (size=163525)
2024-11-20T06:50:04,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741873_1049 (size=163525)
2024-11-20T06:50:04,896 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/65218d679da9421c9941060467392698 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/65218d679da9421c9941060467392698
2024-11-20T06:50:04,903 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 65218d679da9421c9941060467392698(size=159.7 K), total size for store is 177.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:04,903 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:04,903 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085404864; duration=0sec
2024-11-20T06:50:04,903 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:04,903 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:05,386 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:05,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:06,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:06,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:06,881 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-20T06:50:06,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/bfd831c970cb47a9837d136f7a8c6801 is 1080, key is row0220/info:/1732085404866/Put/seqid=0
2024-11-20T06:50:06,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741874_1050 (size=12523)
2024-11-20T06:50:06,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741874_1050 (size=12523)
2024-11-20T06:50:06,904 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/bfd831c970cb47a9837d136f7a8c6801
2024-11-20T06:50:06,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/bfd831c970cb47a9837d136f7a8c6801 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801
2024-11-20T06:50:06,916 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801, entries=7, sequenceid=302, filesize=12.2 K
2024-11-20T06:50:06,918 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for b6811eaddd93e7ae02da7b00eebbc02f in 36ms, sequenceid=302, compaction requested=true
2024-11-20T06:50:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:06,918 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:06,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:06,918 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-11-20T06:50:06,919 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 193966 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:06,919 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:06,919 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:06,919 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/65218d679da9421c9941060467392698, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=189.4 K
2024-11-20T06:50:06,920 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65218d679da9421c9941060467392698, keycount=146, bloomtype=ROW, size=159.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732085384448
2024-11-20T06:50:06,920 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80153bd122d74ef1ab726189dfd21208, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732085404839
2024-11-20T06:50:06,920 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd831c970cb47a9837d136f7a8c6801, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732085404866
2024-11-20T06:50:06,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/806f9eb41e7c446aa85dc29e96d105a6 is 1080, key is row0227/info:/1732085406882/Put/seqid=0
2024-11-20T06:50:06,936 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#91 average throughput is 56.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:06,936 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/48aac6595bd549a5b95e77ef544cf73a is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:06,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741876_1052 (size=184132)
2024-11-20T06:50:06,947 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741876_1052 (size=184132)
2024-11-20T06:50:06,953 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/48aac6595bd549a5b95e77ef544cf73a as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/48aac6595bd549a5b95e77ef544cf73a
2024-11-20T06:50:06,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741875_1051 (size=24412)
2024-11-20T06:50:06,957 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741875_1051 (size=24412)
2024-11-20T06:50:06,958 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/806f9eb41e7c446aa85dc29e96d105a6
2024-11-20T06:50:06,958 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-11-20T06:50:06,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:53268 deadline: 1732085416958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045
2024-11-20T06:50:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., hostname=61c2066a8a0f,40669,1732085371045, seqNum=131 , the old value is region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., hostname=61c2066a8a0f,40669,1732085371045, seqNum=131, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T06:50:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., hostname=61c2066a8a0f,40669,1732085371045, seqNum=131 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=b6811eaddd93e7ae02da7b00eebbc02f, server=61c2066a8a0f,40669,1732085371045
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-20T06:50:06,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., hostname=61c2066a8a0f,40669,1732085371045, seqNum=131 because the exception is null or not the one we care about
2024-11-20T06:50:06,962 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 48aac6595bd549a5b95e77ef544cf73a(size=179.8 K), total size for store is 179.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:06,962 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:06,962 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085406918; duration=0sec
2024-11-20T06:50:06,962 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:06,962 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:06,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/806f9eb41e7c446aa85dc29e96d105a6 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6
2024-11-20T06:50:06,973 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6, entries=18, sequenceid=323, filesize=23.8 K
2024-11-20T06:50:06,974 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for b6811eaddd93e7ae02da7b00eebbc02f in 56ms, sequenceid=323, compaction requested=false
2024-11-20T06:50:06,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:07,387 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:07,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:08,388 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:08,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:09,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:09,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:10,389 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:10,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:11,390 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:11,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:12,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:12,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:12,781 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-20T06:50:12,781 INFO  [master/61c2066a8a0f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-20T06:50:13,391 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:13,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:14,392 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:14,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:15,393 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:15,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:16,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:16,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:17,015 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-11-20T06:50:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40669 {}] regionserver.HRegion(8855): Flush requested on b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:17,032 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b6811eaddd93e7ae02da7b00eebbc02f 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-20T06:50:17,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/64cb14f5e7e04eaf8808627e145646d1 is 1080, key is row0245/info:/1732085406926/Put/seqid=0
2024-11-20T06:50:17,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741877_1053 (size=17918)
2024-11-20T06:50:17,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741877_1053 (size=17918)
2024-11-20T06:50:17,041 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/64cb14f5e7e04eaf8808627e145646d1
2024-11-20T06:50:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/64cb14f5e7e04eaf8808627e145646d1 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1
2024-11-20T06:50:17,050 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1, entries=12, sequenceid=339, filesize=17.5 K
2024-11-20T06:50:17,051 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for b6811eaddd93e7ae02da7b00eebbc02f in 19ms, sequenceid=339, compaction requested=true
2024-11-20T06:50:17,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:17,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b6811eaddd93e7ae02da7b00eebbc02f:info, priority=-2147483648, current under compaction store size is 1
2024-11-20T06:50:17,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:17,051 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-20T06:50:17,052 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 226462 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-20T06:50:17,052 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1541): b6811eaddd93e7ae02da7b00eebbc02f/info is initiating minor compaction (all files)
2024-11-20T06:50:17,052 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b6811eaddd93e7ae02da7b00eebbc02f/info in TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:17,053 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/48aac6595bd549a5b95e77ef544cf73a, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1] into tmpdir=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp, totalSize=221.2 K
2024-11-20T06:50:17,053 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48aac6595bd549a5b95e77ef544cf73a, keycount=165, bloomtype=ROW, size=179.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732085384448
2024-11-20T06:50:17,053 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806f9eb41e7c446aa85dc29e96d105a6, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732085406882
2024-11-20T06:50:17,054 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64cb14f5e7e04eaf8808627e145646d1, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732085406926
2024-11-20T06:50:17,071 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b6811eaddd93e7ae02da7b00eebbc02f#info#compaction#93 average throughput is 50.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-20T06:50:17,071 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/9dffc21cff844a93825d758e19d2b251 is 1080, key is row0062/info:/1732085384448/Put/seqid=0
2024-11-20T06:50:17,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741878_1054 (size=216681)
2024-11-20T06:50:17,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741878_1054 (size=216681)
2024-11-20T06:50:17,081 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/.tmp/info/9dffc21cff844a93825d758e19d2b251 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/9dffc21cff844a93825d758e19d2b251
2024-11-20T06:50:17,087 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b6811eaddd93e7ae02da7b00eebbc02f/info of b6811eaddd93e7ae02da7b00eebbc02f into 9dffc21cff844a93825d758e19d2b251(size=211.6 K), total size for store is 211.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-20T06:50:17,087 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b6811eaddd93e7ae02da7b00eebbc02f:

2024-11-20T06:50:17,087 INFO  [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., storeName=b6811eaddd93e7ae02da7b00eebbc02f/info, priority=13, startTime=1732085417051; duration=0sec
2024-11-20T06:50:17,087 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-20T06:50:17,087 DEBUG [RS:0;61c2066a8a0f:40669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b6811eaddd93e7ae02da7b00eebbc02f:info
2024-11-20T06:50:17,394 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:17,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:18,395 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:18,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:19,033 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-20T06:50:19,033 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C40669%2C1732085371045.1732085419033
2024-11-20T06:50:19,039 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,039 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,039 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,040 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,040 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,040 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045/61c2066a8a0f%2C40669%2C1732085371045.1732085371557 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045/61c2066a8a0f%2C40669%2C1732085371045.1732085419033
2024-11-20T06:50:19,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741833_1009 (size=317837)
2024-11-20T06:50:19,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741833_1009 (size=317837)
2024-11-20T06:50:19,055 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40257:40257),(127.0.0.1/127.0.0.1:44703:44703)]
2024-11-20T06:50:19,056 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/WALs/61c2066a8a0f,40669,1732085371045/61c2066a8a0f%2C40669%2C1732085371045.1732085371557 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/oldWALs/61c2066a8a0f%2C40669%2C1732085371045.1732085371557
2024-11-20T06:50:19,060 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:50:19,060 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:50:19,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:19,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:19,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:19,061 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:50:19,061 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:50:19,061 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=425487258, stopped=false
2024-11-20T06:50:19,061 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,38673,1732085370902
2024-11-20T06:50:19,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:19,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:19,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:19,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:19,108 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:50:19,109 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:50:19,109 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:19,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:19,109 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,40669,1732085371045' *****
2024-11-20T06:50:19,109 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:50:19,109 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:50:19,109 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:50:19,109 INFO  [RS:0;61c2066a8a0f:40669 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:50:19,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(3091): Received CLOSE for b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(3091): Received CLOSE for 0c3c5b16ba3ecc9dc62bd758dd3a7764
2024-11-20T06:50:19,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,40669,1732085371045
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:40669.
2024-11-20T06:50:19,110 DEBUG [RS:0;61c2066a8a0f:40669 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:19,110 DEBUG [RS:0;61c2066a8a0f:40669 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:50:19,110 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:50:19,111 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b6811eaddd93e7ae02da7b00eebbc02f, disabling compactions & flushes
2024-11-20T06:50:19,111 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1325): Online Regions={b6811eaddd93e7ae02da7b00eebbc02f=TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f., 0c3c5b16ba3ecc9dc62bd758dd3a7764=TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764., 1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:50:19,111 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:19,111 DEBUG [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1351): Waiting on 0c3c5b16ba3ecc9dc62bd758dd3a7764, 1588230740, b6811eaddd93e7ae02da7b00eebbc02f
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f. after waiting 0 ms
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:19,111 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:50:19,111 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:50:19,111 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-20T06:50:19,111 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-top, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/e5dd2998a97841d48f9e0cc9e5d5c729, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8ff4dac7b63e4ff0aab8bf622874e9ac, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/25a82300ffee4e029459fc6f8d991012, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/21e4b137015a41fe93c7fda9cbf55d13, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/a45a020f40bd40ab9113a1b083005374, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/65218d679da9421c9941060467392698, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/48aac6595bd549a5b95e77ef544cf73a, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1] to archive
2024-11-20T06:50:19,112 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:50:19,114 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97
2024-11-20T06:50:19,116 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/9b4d15d45f9e4118bef3a5151c911f6b is 186, key is TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764./info:regioninfo/1732085387326/Put/seqid=0
2024-11-20T06:50:19,116 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-ac6b0104d658482983f365c501d2231c
2024-11-20T06:50:19,121 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/e5dd2998a97841d48f9e0cc9e5d5c729 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/e5dd2998a97841d48f9e0cc9e5d5c729
2024-11-20T06:50:19,124 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/TestLogRolling-testLogRolling=4a0951ded7cf07938c682cc129284f97-e1f611fb594a43488e8ce8a3015c0aa4
2024-11-20T06:50:19,126 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e9dbdf908a45ca9f417e839de663a4
2024-11-20T06:50:19,127 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8ff4dac7b63e4ff0aab8bf622874e9ac to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8ff4dac7b63e4ff0aab8bf622874e9ac
2024-11-20T06:50:19,129 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1cef473351b14b518162bcc72ee07c1d
2024-11-20T06:50:19,130 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ffc78c591cf744509140f1d0c7f9415f
2024-11-20T06:50:19,132 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/25a82300ffee4e029459fc6f8d991012 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/25a82300ffee4e029459fc6f8d991012
2024-11-20T06:50:19,134 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/8005e9dd5c4f49c5ab7dff7723ba9540
2024-11-20T06:50:19,135 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/ecefddc745634deaaafd5ea0be3b29ee
2024-11-20T06:50:19,137 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/21e4b137015a41fe93c7fda9cbf55d13 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/21e4b137015a41fe93c7fda9cbf55d13
2024-11-20T06:50:19,138 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/c42e7575f1db452ab7e30fa883faba42
2024-11-20T06:50:19,140 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/f5e217e508214694803e1c5adb9ff48f
2024-11-20T06:50:19,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741880_1056 (size=6153)
2024-11-20T06:50:19,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741880_1056 (size=6153)
2024-11-20T06:50:19,142 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/a45a020f40bd40ab9113a1b083005374 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/a45a020f40bd40ab9113a1b083005374
2024-11-20T06:50:19,142 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/9b4d15d45f9e4118bef3a5151c911f6b
2024-11-20T06:50:19,143 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/7e88bf5798e44a0dae96273ce0b2ecb0
2024-11-20T06:50:19,145 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/0bc38befba24455e8823fbc7fb297c9d
2024-11-20T06:50:19,146 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/65218d679da9421c9941060467392698 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/65218d679da9421c9941060467392698
2024-11-20T06:50:19,147 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/1b1c217bc62f4709a27c7949b6d858e5
2024-11-20T06:50:19,148 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/.tmp/info/9b4d15d45f9e4118bef3a5151c911f6b as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/info/9b4d15d45f9e4118bef3a5151c911f6b
2024-11-20T06:50:19,149 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/80153bd122d74ef1ab726189dfd21208
2024-11-20T06:50:19,154 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/48aac6595bd549a5b95e77ef544cf73a to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/48aac6595bd549a5b95e77ef544cf73a
2024-11-20T06:50:19,164 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/info/9b4d15d45f9e4118bef3a5151c911f6b, entries=5, sequenceid=21, filesize=6.0 K
2024-11-20T06:50:19,165 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/bfd831c970cb47a9837d136f7a8c6801
2024-11-20T06:50:19,166 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=21, compaction requested=false
2024-11-20T06:50:19,171 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/806f9eb41e7c446aa85dc29e96d105a6
2024-11-20T06:50:19,174 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/info/64cb14f5e7e04eaf8808627e145646d1
2024-11-20T06:50:19,175 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=61c2066a8a0f:38673 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-20T06:50:19,175 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [e5dd2998a97841d48f9e0cc9e5d5c729=42984, f5e9dbdf908a45ca9f417e839de663a4=12516, 8ff4dac7b63e4ff0aab8bf622874e9ac=67948, 1cef473351b14b518162bcc72ee07c1d=22238, ffc78c591cf744509140f1d0c7f9415f=15750, 25a82300ffee4e029459fc6f8d991012=88609, 8005e9dd5c4f49c5ab7dff7723ba9540=14672, ecefddc745634deaaafd5ea0be3b29ee=15750, 21e4b137015a41fe93c7fda9cbf55d13=113509, c42e7575f1db452ab7e30fa883faba42=19000, f5e217e508214694803e1c5adb9ff48f=16828, a45a020f40bd40ab9113a1b083005374=144000, 7e88bf5798e44a0dae96273ce0b2ecb0=23316, 0bc38befba24455e8823fbc7fb297c9d=12520, 65218d679da9421c9941060467392698=163525, 1b1c217bc62f4709a27c7949b6d858e5=16839, 80153bd122d74ef1ab726189dfd21208=17918, 48aac6595bd549a5b95e77ef544cf73a=184132, bfd831c970cb47a9837d136f7a8c6801=12523, 806f9eb41e7c446aa85dc29e96d105a6=24412, 64cb14f5e7e04eaf8808627e145646d1=17918]
2024-11-20T06:50:19,187 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/b6811eaddd93e7ae02da7b00eebbc02f/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=130
2024-11-20T06:50:19,188 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:19,188 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b6811eaddd93e7ae02da7b00eebbc02f:
Waiting for close lock at 1732085419110Running coprocessor pre-close hooks at 1732085419110Disabling compacts and flushes for region at 1732085419110Disabling writes for close at 1732085419111 (+1 ms)Writing region close event to WAL at 1732085419179 (+68 ms)Running coprocessor post-close hooks at 1732085419188 (+9 ms)Closed at 1732085419188
2024-11-20T06:50:19,188 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732085386563.b6811eaddd93e7ae02da7b00eebbc02f.
2024-11-20T06:50:19,189 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0c3c5b16ba3ecc9dc62bd758dd3a7764, disabling compactions & flushes
2024-11-20T06:50:19,189 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:50:19,189 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:50:19,189 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764. after waiting 0 ms
2024-11-20T06:50:19,189 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:50:19,191 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97->hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/4a0951ded7cf07938c682cc129284f97/info/375092727ba4409f9acbe0cdf65d0fac-bottom] to archive
2024-11-20T06:50:19,192 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-20T06:50:19,210 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97 to hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/archive/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/info/375092727ba4409f9acbe0cdf65d0fac.4a0951ded7cf07938c682cc129284f97
2024-11-20T06:50:19,210 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-20T06:50:19,219 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-20T06:50:19,219 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:50:19,219 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:50:19,220 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085419111Running coprocessor pre-close hooks at 1732085419111Disabling compacts and flushes for region at 1732085419111Disabling writes for close at 1732085419111Obtaining lock to block concurrent updates at 1732085419111Preparing flush snapshotting stores in 1588230740 at 1732085419111Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=670, getHeapSize=2000, getOffHeapSize=0, getCellsCount=5 at 1732085419112 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085419113 (+1 ms)Flushing 1588230740/info: creating writer at 1732085419113Flushing 1588230740/info: appending metadata at 1732085419116 (+3 ms)Flushing 1588230740/info: closing flushed file at 1732085419116Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2195bb03: reopening flushed file at 1732085419147 (+31 ms)Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=21, compaction requested=false at 1732085419166 (+19 ms)Writing region close event to WAL at 1732085419192 (+26 ms)Running coprocessor post-close hooks at 1732085419219 (+27 ms)Closed at 1732085419219
2024-11-20T06:50:19,220 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:50:19,223 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/data/default/TestLogRolling-testLogRolling/0c3c5b16ba3ecc9dc62bd758dd3a7764/recovered.edits/134.seqid, newMaxSeqId=134, maxSeqId=130
2024-11-20T06:50:19,223 INFO  [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:50:19,224 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0c3c5b16ba3ecc9dc62bd758dd3a7764:
Waiting for close lock at 1732085419189Running coprocessor pre-close hooks at 1732085419189Disabling compacts and flushes for region at 1732085419189Disabling writes for close at 1732085419189Writing region close event to WAL at 1732085419216 (+27 ms)Running coprocessor post-close hooks at 1732085419223 (+7 ms)Closed at 1732085419223
2024-11-20T06:50:19,224 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732085386563.0c3c5b16ba3ecc9dc62bd758dd3a7764.
2024-11-20T06:50:19,311 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,40669,1732085371045; all regions closed.
2024-11-20T06:50:19,311 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,311 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,311 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,312 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,312 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741834_1010 (size=8107)
2024-11-20T06:50:19,314 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741834_1010 (size=8107)
2024-11-20T06:50:19,316 DEBUG [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/oldWALs
2024-11-20T06:50:19,316 INFO  [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C40669%2C1732085371045.meta:.meta(num 1732085372007)
2024-11-20T06:50:19,317 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,317 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,317 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,317 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,317 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741879_1055 (size=780)
2024-11-20T06:50:19,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741879_1055 (size=780)
2024-11-20T06:50:19,320 DEBUG [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/oldWALs
2024-11-20T06:50:19,320 INFO  [RS:0;61c2066a8a0f:40669 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C40669%2C1732085371045:(num 1732085419033)
2024-11-20T06:50:19,320 DEBUG [RS:0;61c2066a8a0f:40669 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:19,321 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:50:19,321 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:50:19,321 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-20T06:50:19,321 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:50:19,321 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:50:19,321 INFO  [RS:0;61c2066a8a0f:40669 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40669
2024-11-20T06:50:19,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,40669,1732085371045
2024-11-20T06:50:19,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:50:19,344 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:50:19,353 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,40669,1732085371045]
2024-11-20T06:50:19,361 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,40669,1732085371045 already deleted, retry=false
2024-11-20T06:50:19,361 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,40669,1732085371045 expired; onlineServers=0
2024-11-20T06:50:19,361 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,38673,1732085370902' *****
2024-11-20T06:50:19,361 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:50:19,361 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:50:19,361 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:50:19,362 DEBUG [M:0;61c2066a8a0f:38673 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:50:19,362 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:50:19,362 DEBUG [M:0;61c2066a8a0f:38673 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:50:19,362 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085371354 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085371354,5,FailOnTimeoutGroup]
2024-11-20T06:50:19,362 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085371351 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085371351,5,FailOnTimeoutGroup]
2024-11-20T06:50:19,362 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:50:19,362 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:50:19,362 DEBUG [M:0;61c2066a8a0f:38673 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:50:19,362 INFO  [M:0;61c2066a8a0f:38673 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:50:19,363 INFO  [M:0;61c2066a8a0f:38673 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:50:19,363 INFO  [M:0;61c2066a8a0f:38673 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:50:19,363 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:50:19,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:50:19,370 DEBUG [M:0;61c2066a8a0f:38673 {}] zookeeper.ZKUtil(347): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:50:19,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:19,370 WARN  [M:0;61c2066a8a0f:38673 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:50:19,370 INFO  [M:0;61c2066a8a0f:38673 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/.lastflushedseqids
2024-11-20T06:50:19,395 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741881_1057 (size=228)
2024-11-20T06:50:19,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741881_1057 (size=228)
2024-11-20T06:50:19,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:19,396 INFO  [M:0;61c2066a8a0f:38673 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:50:19,397 INFO  [M:0;61c2066a8a0f:38673 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:50:19,397 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:50:19,397 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:19,397 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:19,397 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:50:19,397 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:19,397 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.36 KB
2024-11-20T06:50:19,424 INFO  [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:50:19,424 DEBUG [M:0;61c2066a8a0f:38673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fae78fc287874f548969f93f698b8801 is 82, key is hbase:meta,,1/info:regioninfo/1732085372032/Put/seqid=0
2024-11-20T06:50:19,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741882_1058 (size=5672)
2024-11-20T06:50:19,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741882_1058 (size=5672)
2024-11-20T06:50:19,433 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fae78fc287874f548969f93f698b8801
2024-11-20T06:50:19,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:19,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40669-0x101584ae2c90001, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:19,453 INFO  [RS:0;61c2066a8a0f:40669 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:50:19,453 INFO  [RS:0;61c2066a8a0f:40669 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,40669,1732085371045; zookeeper connection closed.
2024-11-20T06:50:19,454 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6416a18d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6416a18d
2024-11-20T06:50:19,454 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:50:19,459 DEBUG [M:0;61c2066a8a0f:38673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a975126a768a4781b5fed8eca4270d32 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085372969/Put/seqid=0
2024-11-20T06:50:19,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741883_1059 (size=7089)
2024-11-20T06:50:19,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741883_1059 (size=7089)
2024-11-20T06:50:19,470 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a975126a768a4781b5fed8eca4270d32
2024-11-20T06:50:19,475 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a975126a768a4781b5fed8eca4270d32
2024-11-20T06:50:19,494 DEBUG [M:0;61c2066a8a0f:38673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3283925f10db42a6b8fd8ec7e24b3736 is 69, key is 61c2066a8a0f,40669,1732085371045/rs:state/1732085371400/Put/seqid=0
2024-11-20T06:50:19,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741884_1060 (size=5156)
2024-11-20T06:50:19,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741884_1060 (size=5156)
2024-11-20T06:50:19,505 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3283925f10db42a6b8fd8ec7e24b3736
2024-11-20T06:50:19,539 DEBUG [M:0;61c2066a8a0f:38673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4559f92bc7484cdbb7bf07069583327c is 52, key is load_balancer_on/state:d/1732085372182/Put/seqid=0
2024-11-20T06:50:19,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741885_1061 (size=5056)
2024-11-20T06:50:19,560 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741885_1061 (size=5056)
2024-11-20T06:50:19,565 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4559f92bc7484cdbb7bf07069583327c
2024-11-20T06:50:19,574 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fae78fc287874f548969f93f698b8801 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fae78fc287874f548969f93f698b8801
2024-11-20T06:50:19,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:19,583 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fae78fc287874f548969f93f698b8801, entries=8, sequenceid=125, filesize=5.5 K
2024-11-20T06:50:19,584 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a975126a768a4781b5fed8eca4270d32 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a975126a768a4781b5fed8eca4270d32
2024-11-20T06:50:19,589 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a975126a768a4781b5fed8eca4270d32
2024-11-20T06:50:19,589 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a975126a768a4781b5fed8eca4270d32, entries=13, sequenceid=125, filesize=6.9 K
2024-11-20T06:50:19,595 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3283925f10db42a6b8fd8ec7e24b3736 as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3283925f10db42a6b8fd8ec7e24b3736
2024-11-20T06:50:19,601 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3283925f10db42a6b8fd8ec7e24b3736, entries=1, sequenceid=125, filesize=5.0 K
2024-11-20T06:50:19,602 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4559f92bc7484cdbb7bf07069583327c as hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4559f92bc7484cdbb7bf07069583327c
2024-11-20T06:50:19,607 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45627/user/jenkins/test-data/3bf5ec69-762e-fc03-57c7-36d42c93d942/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4559f92bc7484cdbb7bf07069583327c, entries=1, sequenceid=125, filesize=4.9 K
2024-11-20T06:50:19,608 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 211ms, sequenceid=125, compaction requested=false
2024-11-20T06:50:19,623 INFO  [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:19,623 DEBUG [M:0;61c2066a8a0f:38673 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085419397Disabling compacts and flushes for region at 1732085419397Disabling writes for close at 1732085419397Obtaining lock to block concurrent updates at 1732085419397Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085419397Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1732085419397Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085419398 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085419398Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085419424 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085419424Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085419439 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085419459 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085419459Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085419475 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085419493 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085419493Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085419516 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085419538 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085419538Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d0e65e9: reopening flushed file at 1732085419573 (+35 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2611edea: reopening flushed file at 1732085419583 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7801eefd: reopening flushed file at 1732085419590 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ce2832: reopening flushed file at 1732085419602 (+12 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 211ms, sequenceid=125, compaction requested=false at 1732085419608 (+6 ms)Writing region close event to WAL at 1732085419623 (+15 ms)Closed at 1732085419623
2024-11-20T06:50:19,625 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,625 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,625 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,625 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,625 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:19,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37317 is added to blk_1073741830_1006 (size=61320)
2024-11-20T06:50:19,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36161 is added to blk_1073741830_1006 (size=61320)
2024-11-20T06:50:19,628 INFO  [M:0;61c2066a8a0f:38673 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:50:19,628 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:50:19,629 INFO  [M:0;61c2066a8a0f:38673 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38673
2024-11-20T06:50:19,629 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:50:19,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:19,737 INFO  [M:0;61c2066a8a0f:38673 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:50:19,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38673-0x101584ae2c90000, quorum=127.0.0.1:57279, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:19,739 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@dcfcbff{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:19,740 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3dc4994c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:19,740 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:19,740 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2fc2e7d1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:19,740 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ab86f9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:19,741 WARN  [BP-101583425-172.17.0.2-1732085369237 heartbeating to localhost/127.0.0.1:45627 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:50:19,741 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:50:19,741 WARN  [BP-101583425-172.17.0.2-1732085369237 heartbeating to localhost/127.0.0.1:45627 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-101583425-172.17.0.2-1732085369237 (Datanode Uuid 9349f7e6-92c7-423b-9349-9a118e77bcbb) service to localhost/127.0.0.1:45627
2024-11-20T06:50:19,741 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:50:19,742 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data3/current/BP-101583425-172.17.0.2-1732085369237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:19,742 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data4/current/BP-101583425-172.17.0.2-1732085369237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:19,742 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:50:19,747 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@631c133{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:19,747 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3523e770{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:19,748 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:19,748 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@60b9b83d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:19,748 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bb23947{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:19,750 WARN  [BP-101583425-172.17.0.2-1732085369237 heartbeating to localhost/127.0.0.1:45627 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:50:19,750 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:50:19,750 WARN  [BP-101583425-172.17.0.2-1732085369237 heartbeating to localhost/127.0.0.1:45627 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-101583425-172.17.0.2-1732085369237 (Datanode Uuid c2f8cf3e-d0bc-4d65-9190-98b6585122bf) service to localhost/127.0.0.1:45627
2024-11-20T06:50:19,750 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:50:19,750 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data1/current/BP-101583425-172.17.0.2-1732085369237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:19,750 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/cluster_6be240b6-fecd-6eb9-8970-77fbd19f1530/data/data2/current/BP-101583425-172.17.0.2-1732085369237 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:19,751 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:50:19,757 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7018a0ae{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:50:19,758 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7305dd28{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:19,758 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:19,758 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75966949{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:19,758 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e36d39c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:19,766 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:50:19,794 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:50:19,807 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 207)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45627
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45627 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45627 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45627
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:45627
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:45627
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:45627 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45627
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=509 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4354 (was 3576) - AvailableMemoryMB LEAK? -
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=509, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=4351
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.log.dir so I do NOT create it in target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9965841b-52ca-dd00-b936-e7541226f10d/hadoop.tmp.dir so I do NOT create it in target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b, deleteOnExit=true
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/test.cache.data in system properties and HBase conf
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.tmp.dir in system properties and HBase conf
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir in system properties and HBase conf
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-20T06:50:19,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-20T06:50:19,819 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/nfs.dump.dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/java.io.tmpdir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-20T06:50:19,819 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-20T06:50:19,831 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:50:20,097 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:50:20,102 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:50:20,103 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:50:20,103 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:50:20,103 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:50:20,107 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:50:20,108 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@246d323a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:50:20,109 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e290644{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:50:20,246 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7043bc8e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/java.io.tmpdir/jetty-localhost-44959-hadoop-hdfs-3_4_1-tests_jar-_-any-1750634922493491358/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:50:20,247 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5eeda3c1{HTTP/1.1, (http/1.1)}{localhost:44959}
2024-11-20T06:50:20,247 INFO  [Time-limited test {}] server.Server(415): Started @303589ms
2024-11-20T06:50:20,265 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-20T06:50:20,396 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:20,476 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:50:20,480 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:50:20,481 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:50:20,481 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:50:20,481 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:50:20,481 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@352a3917{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:50:20,482 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9090c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:50:20,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:20,585 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12ac7c6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/java.io.tmpdir/jetty-localhost-46833-hadoop-hdfs-3_4_1-tests_jar-_-any-12454352263879438346/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:20,586 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9268180{HTTP/1.1, (http/1.1)}{localhost:46833}
2024-11-20T06:50:20,586 INFO  [Time-limited test {}] server.Server(415): Started @303928ms
2024-11-20T06:50:20,587 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:50:20,623 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-20T06:50:20,626 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-20T06:50:20,627 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-20T06:50:20,627 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-20T06:50:20,627 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-20T06:50:20,627 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@392cf3b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,AVAILABLE}
2024-11-20T06:50:20,628 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a1af98b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-20T06:50:20,752 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a9a5fda{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/java.io.tmpdir/jetty-localhost-36657-hadoop-hdfs-3_4_1-tests_jar-_-any-12122256777745193298/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:20,753 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5eea892a{HTTP/1.1, (http/1.1)}{localhost:36657}
2024-11-20T06:50:20,753 INFO  [Time-limited test {}] server.Server(415): Started @304095ms
2024-11-20T06:50:20,754 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-20T06:50:21,247 WARN  [Thread-2512 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data1/current/BP-1480802988-172.17.0.2-1732085419835/current, will proceed with Du for space computation calculation, 
2024-11-20T06:50:21,247 WARN  [Thread-2513 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data2/current/BP-1480802988-172.17.0.2-1732085419835/current, will proceed with Du for space computation calculation, 
2024-11-20T06:50:21,271 WARN  [Thread-2477 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:50:21,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88a5e106c7d01b8d with lease ID 0xd3995610b28bfc28: Processing first storage report for DS-d61dc5b1-74fb-41ab-ae02-30d4ed9cbd37 from datanode DatanodeRegistration(127.0.0.1:32907, datanodeUuid=14814c17-3547-4d58-9685-864590542ba5, infoPort=41363, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835)
2024-11-20T06:50:21,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88a5e106c7d01b8d with lease ID 0xd3995610b28bfc28: from storage DS-d61dc5b1-74fb-41ab-ae02-30d4ed9cbd37 node DatanodeRegistration(127.0.0.1:32907, datanodeUuid=14814c17-3547-4d58-9685-864590542ba5, infoPort=41363, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-20T06:50:21,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x88a5e106c7d01b8d with lease ID 0xd3995610b28bfc28: Processing first storage report for DS-ff16621f-e1b2-40b1-ace3-823588c04cc4 from datanode DatanodeRegistration(127.0.0.1:32907, datanodeUuid=14814c17-3547-4d58-9685-864590542ba5, infoPort=41363, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835)
2024-11-20T06:50:21,273 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x88a5e106c7d01b8d with lease ID 0xd3995610b28bfc28: from storage DS-ff16621f-e1b2-40b1-ace3-823588c04cc4 node DatanodeRegistration(127.0.0.1:32907, datanodeUuid=14814c17-3547-4d58-9685-864590542ba5, infoPort=41363, infoSecurePort=0, ipcPort=41811, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:50:21,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:21,458 WARN  [Thread-2525 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data3/current/BP-1480802988-172.17.0.2-1732085419835/current, will proceed with Du for space computation calculation, 
2024-11-20T06:50:21,458 WARN  [Thread-2526 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data4/current/BP-1480802988-172.17.0.2-1732085419835/current, will proceed with Du for space computation calculation, 
2024-11-20T06:50:21,478 WARN  [Thread-2500 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-20T06:50:21,480 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e2998ee8674bc66 with lease ID 0xd3995610b28bfc29: Processing first storage report for DS-55a23047-d83e-4e29-a08b-072252a5369a from datanode DatanodeRegistration(127.0.0.1:45141, datanodeUuid=7c8f28ff-91f6-4e8d-a143-0e077de0c49e, infoPort=40047, infoSecurePort=0, ipcPort=44387, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835)
2024-11-20T06:50:21,480 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e2998ee8674bc66 with lease ID 0xd3995610b28bfc29: from storage DS-55a23047-d83e-4e29-a08b-072252a5369a node DatanodeRegistration(127.0.0.1:45141, datanodeUuid=7c8f28ff-91f6-4e8d-a143-0e077de0c49e, infoPort=40047, infoSecurePort=0, ipcPort=44387, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:50:21,480 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e2998ee8674bc66 with lease ID 0xd3995610b28bfc29: Processing first storage report for DS-a409058a-fe78-42a5-9e03-c45d76dc5b94 from datanode DatanodeRegistration(127.0.0.1:45141, datanodeUuid=7c8f28ff-91f6-4e8d-a143-0e077de0c49e, infoPort=40047, infoSecurePort=0, ipcPort=44387, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835)
2024-11-20T06:50:21,480 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e2998ee8674bc66 with lease ID 0xd3995610b28bfc29: from storage DS-a409058a-fe78-42a5-9e03-c45d76dc5b94 node DatanodeRegistration(127.0.0.1:45141, datanodeUuid=7c8f28ff-91f6-4e8d-a143-0e077de0c49e, infoPort=40047, infoSecurePort=0, ipcPort=44387, storageInfo=lv=-57;cid=testClusterID;nsid=1459188942;c=1732085419835), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-20T06:50:21,496 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827
2024-11-20T06:50:21,499 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/zookeeper_0, clientPort=63817, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-20T06:50:21,499 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63817
2024-11-20T06:50:21,500 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,501 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:50:21,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741825_1001 (size=7)
2024-11-20T06:50:21,514 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f with version=8
2024-11-20T06:50:21,514 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33799/user/jenkins/test-data/588e991b-9ede-26f7-173b-e865bb17fb2d/hbase-staging
2024-11-20T06:50:21,516 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-20T06:50:21,517 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:50:21,518 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42629
2024-11-20T06:50:21,520 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42629 connecting to ZooKeeper ensemble=127.0.0.1:63817
2024-11-20T06:50:21,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426290x0, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:50:21,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42629-0x101584ba87f0000 connected
2024-11-20T06:50:21,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:21,628 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,630 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:21,632 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f, hbase.cluster.distributed=false
2024-11-20T06:50:21,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:50:21,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42629
2024-11-20T06:50:21,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42629
2024-11-20T06:50:21,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42629
2024-11-20T06:50:21,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42629
2024-11-20T06:50:21,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42629
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-20T06:50:21,656 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-20T06:50:21,658 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45189
2024-11-20T06:50:21,659 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45189 connecting to ZooKeeper ensemble=127.0.0.1:63817
2024-11-20T06:50:21,659 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,661 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451890x0, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-20T06:50:21,670 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451890x0, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:21,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45189-0x101584ba87f0001 connected
2024-11-20T06:50:21,670 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-20T06:50:21,671 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-20T06:50:21,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-20T06:50:21,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-20T06:50:21,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45189
2024-11-20T06:50:21,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45189
2024-11-20T06:50:21,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45189
2024-11-20T06:50:21,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45189
2024-11-20T06:50:21,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45189
2024-11-20T06:50:21,684 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:42629
2024-11-20T06:50:21,685 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:50:21,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:50:21,695 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-20T06:50:21,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,703 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-20T06:50:21,703 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,42629,1732085421516 from backup master directory
2024-11-20T06:50:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:50:21,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-20T06:50:21,711 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:50:21,711 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,714 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/hbase.id] with ID: fdf0305b-36c7-467f-90c8-9a9dc96e6ba9
2024-11-20T06:50:21,714 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/.tmp/hbase.id
2024-11-20T06:50:21,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:50:21,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741826_1002 (size=42)
2024-11-20T06:50:21,720 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/.tmp/hbase.id]:[hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/hbase.id]
2024-11-20T06:50:21,736 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:21,737 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-20T06:50:21,738 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-20T06:50:21,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:50:21,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741827_1003 (size=196)
2024-11-20T06:50:21,777 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-20T06:50:21,778 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-20T06:50:21,778 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:50:21,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:50:21,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741828_1004 (size=1189)
2024-11-20T06:50:21,786 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store
2024-11-20T06:50:21,803 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:50:21,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741829_1005 (size=34)
2024-11-20T06:50:21,804 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:50:21,804 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:50:21,804 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:21,805 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:21,805 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:50:21,805 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:21,805 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:21,805 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085421804Disabling compacts and flushes for region at 1732085421804Disabling writes for close at 1732085421805 (+1 ms)Writing region close event to WAL at 1732085421805Closed at 1732085421805
2024-11-20T06:50:21,806 WARN  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/.initializing
2024-11-20T06:50:21,806 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/WALs/61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,808 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C42629%2C1732085421516, suffix=, logDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/WALs/61c2066a8a0f,42629,1732085421516, archiveDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/oldWALs, maxLogs=10
2024-11-20T06:50:21,809 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C42629%2C1732085421516.1732085421809
2024-11-20T06:50:21,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/WALs/61c2066a8a0f,42629,1732085421516/61c2066a8a0f%2C42629%2C1732085421516.1732085421809
2024-11-20T06:50:21,820 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:40047:40047)]
2024-11-20T06:50:21,821 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:50:21,821 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:50:21,821 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,821 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,822 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:50:21,823 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-20T06:50:21,824 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:21,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:21,824 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:50:21,825 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-20T06:50:21,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:21,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:50:21,826 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:50:21,827 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-20T06:50:21,827 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:21,828 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:50:21,828 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-20T06:50:21,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-20T06:50:21,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:21,829 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-20T06:50:21,829 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,830 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,830 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,831 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,831 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,832 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-20T06:50:21,833 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-20T06:50:21,835 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:50:21,835 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805935, jitterRate=0.02479994297027588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-20T06:50:21,836 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732085421821Initializing all the Stores at 1732085421822 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085421822Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085421822Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085421822Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085421822Cleaning up temporary data from old regions at 1732085421831 (+9 ms)Region opened successfully at 1732085421836 (+5 ms)
2024-11-20T06:50:21,836 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-20T06:50:21,839 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e4924e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:50:21,840 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-20T06:50:21,840 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-20T06:50:21,840 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-20T06:50:21,840 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-20T06:50:21,841 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-20T06:50:21,841 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-20T06:50:21,841 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-20T06:50:21,857 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-20T06:50:21,858 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-20T06:50:21,877 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-20T06:50:21,878 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-20T06:50:21,880 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-20T06:50:21,887 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-20T06:50:21,888 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-20T06:50:21,894 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-20T06:50:21,902 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-20T06:50:21,904 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-20T06:50:21,911 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-20T06:50:21,915 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-20T06:50:21,919 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-20T06:50:21,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:21,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,929 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,42629,1732085421516, sessionid=0x101584ba87f0000, setting cluster-up flag (Was=false)
2024-11-20T06:50:21,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:21,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,977 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-20T06:50:21,979 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:21,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:21,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:22,019 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-20T06:50:22,020 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:22,035 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-20T06:50:22,039 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-20T06:50:22,040 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-20T06:50:22,040 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-20T06:50:22,040 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,42629,1732085421516
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:50:22,042 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,047 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:50:22,047 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-20T06:50:22,048 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,048 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085452055
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-20T06:50:22,055 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,056 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-20T06:50:22,056 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-20T06:50:22,056 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-20T06:50:22,056 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-20T06:50:22,057 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-20T06:50:22,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085422057,5,FailOnTimeoutGroup]
2024-11-20T06:50:22,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085422057,5,FailOnTimeoutGroup]
2024-11-20T06:50:22,057 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,057 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-20T06:50:22,057 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,057 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:50:22,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741831_1007 (size=1321)
2024-11-20T06:50:22,060 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-20T06:50:22,060 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f
2024-11-20T06:50:22,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:50:22,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741832_1008 (size=32)
2024-11-20T06:50:22,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:50:22,075 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(746): ClusterId : fdf0305b-36c7-467f-90c8-9a9dc96e6ba9
2024-11-20T06:50:22,075 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-20T06:50:22,076 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:50:22,077 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:50:22,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:50:22,080 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:50:22,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:50:22,082 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:50:22,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,083 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,083 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:50:22,084 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:50:22,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,085 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:50:22,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740
2024-11-20T06:50:22,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740
2024-11-20T06:50:22,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:50:22,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:50:22,087 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:50:22,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:50:22,090 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-20T06:50:22,091 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760029, jitterRate=-0.03357437252998352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:50:22,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732085422074Initializing all the Stores at 1732085422075 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422075Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422076 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085422076Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422076Cleaning up temporary data from old regions at 1732085422087 (+11 ms)Region opened successfully at 1732085422091 (+4 ms)
2024-11-20T06:50:22,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:50:22,091 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:50:22,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:50:22,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:50:22,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:50:22,092 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:50:22,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085422091Disabling compacts and flushes for region at 1732085422091Disabling writes for close at 1732085422091Writing region close event to WAL at 1732085422092 (+1 ms)Closed at 1732085422092
2024-11-20T06:50:22,093 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:50:22,093 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-20T06:50:22,093 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-20T06:50:22,094 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:50:22,095 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-20T06:50:22,111 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-20T06:50:22,112 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-20T06:50:22,120 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-20T06:50:22,120 DEBUG [RS:0;61c2066a8a0f:45189 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227cf0bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0
2024-11-20T06:50:22,133 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:45189
2024-11-20T06:50:22,133 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-20T06:50:22,133 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-20T06:50:22,133 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-20T06:50:22,134 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,42629,1732085421516 with port=45189, startcode=1732085421655
2024-11-20T06:50:22,134 DEBUG [RS:0;61c2066a8a0f:45189 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-20T06:50:22,136 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52597, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-20T06:50:22,136 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,136 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42629 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,138 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f
2024-11-20T06:50:22,138 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39117
2024-11-20T06:50:22,138 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-20T06:50:22,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:50:22,144 DEBUG [RS:0;61c2066a8a0f:45189 {}] zookeeper.ZKUtil(111): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,144 WARN  [RS:0;61c2066a8a0f:45189 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-20T06:50:22,144 INFO  [RS:0;61c2066a8a0f:45189 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:50:22,145 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,145 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,45189,1732085421655]
2024-11-20T06:50:22,148 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-20T06:50:22,150 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-20T06:50:22,151 INFO  [RS:0;61c2066a8a0f:45189 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-20T06:50:22,151 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,151 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-20T06:50:22,152 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-20T06:50:22,152 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:50:22,153 DEBUG [RS:0;61c2066a8a0f:45189 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,154 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,45189,1732085421655-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:50:22,176 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-20T06:50:22,176 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,45189,1732085421655-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,176 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,176 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.Replication(171): 61c2066a8a0f,45189,1732085421655 started
2024-11-20T06:50:22,198 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,198 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,45189,1732085421655, RpcServer on 61c2066a8a0f/172.17.0.2:45189, sessionid=0x101584ba87f0001
2024-11-20T06:50:22,199 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-20T06:50:22,199 DEBUG [RS:0;61c2066a8a0f:45189 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,199 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,45189,1732085421655'
2024-11-20T06:50:22,199 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,45189,1732085421655'
2024-11-20T06:50:22,200 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-20T06:50:22,201 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-20T06:50:22,201 DEBUG [RS:0;61c2066a8a0f:45189 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-20T06:50:22,201 INFO  [RS:0;61c2066a8a0f:45189 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-20T06:50:22,201 INFO  [RS:0;61c2066a8a0f:45189 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-20T06:50:22,245 WARN  [61c2066a8a0f:42629 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-20T06:50:22,303 INFO  [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C45189%2C1732085421655, suffix=, logDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/61c2066a8a0f,45189,1732085421655, archiveDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs, maxLogs=32
2024-11-20T06:50:22,304 INFO  [RS:0;61c2066a8a0f:45189 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C45189%2C1732085421655.1732085422304
2024-11-20T06:50:22,314 INFO  [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/61c2066a8a0f,45189,1732085421655/61c2066a8a0f%2C45189%2C1732085421655.1732085422304
2024-11-20T06:50:22,315 DEBUG [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:40047:40047)]
2024-11-20T06:50:22,397 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:22,495 DEBUG [61c2066a8a0f:42629 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-20T06:50:22,496 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,497 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,45189,1732085421655, state=OPENING
2024-11-20T06:50:22,535 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-20T06:50:22,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:22,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:22,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:50:22,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:50:22,561 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-20T06:50:22,561 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,45189,1732085421655}]
2024-11-20T06:50:22,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.1732085227723
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:22,714 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-20T06:50:22,716 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43915, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-20T06:50:22,721 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-20T06:50:22,721 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:50:22,723 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C45189%2C1732085421655.meta, suffix=.meta, logDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/61c2066a8a0f,45189,1732085421655, archiveDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs, maxLogs=32
2024-11-20T06:50:22,724 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 61c2066a8a0f%2C45189%2C1732085421655.meta.1732085422724.meta
2024-11-20T06:50:22,731 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/61c2066a8a0f,45189,1732085421655/61c2066a8a0f%2C45189%2C1732085421655.meta.1732085422724.meta
2024-11-20T06:50:22,734 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40047:40047),(127.0.0.1/127.0.0.1:41363:41363)]
2024-11-20T06:50:22,734 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-20T06:50:22,735 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-20T06:50:22,735 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-20T06:50:22,736 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-20T06:50:22,737 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-20T06:50:22,737 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,738 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,738 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-20T06:50:22,738 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-20T06:50:22,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,738 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,738 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-20T06:50:22,739 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-20T06:50:22,739 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,739 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,739 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-20T06:50:22,740 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-20T06:50:22,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-20T06:50:22,740 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-20T06:50:22,740 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-20T06:50:22,741 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740
2024-11-20T06:50:22,742 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740
2024-11-20T06:50:22,743 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-20T06:50:22,743 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-20T06:50:22,743 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-20T06:50:22,744 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-20T06:50:22,745 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799207, jitterRate=0.016245156526565552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-20T06:50:22,745 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-20T06:50:22,745 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732085422735Writing region info on filesystem at 1732085422735Initializing all the Stores at 1732085422736 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422736Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422736Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085422736Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085422736Cleaning up temporary data from old regions at 1732085422743 (+7 ms)Running coprocessor post-open hooks at 1732085422745 (+2 ms)Region opened successfully at 1732085422745
2024-11-20T06:50:22,746 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085422713
2024-11-20T06:50:22,748 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-20T06:50:22,748 INFO  [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-20T06:50:22,749 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,750 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,45189,1732085421655, state=OPEN
2024-11-20T06:50:22,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:50:22,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-20T06:50:22,801 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:50:22,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-20T06:50:22,804 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-20T06:50:22,804 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,45189,1732085421655 in 240 msec
2024-11-20T06:50:22,806 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-20T06:50:22,806 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 711 msec
2024-11-20T06:50:22,807 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-20T06:50:22,807 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-20T06:50:22,808 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:50:22,808 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,45189,1732085421655, seqNum=-1]
2024-11-20T06:50:22,808 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:50:22,809 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36527, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:50:22,814 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 774 msec
2024-11-20T06:50:22,814 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085422814, completionTime=-1
2024-11-20T06:50:22,814 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-20T06:50:22,814 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-20T06:50:22,815 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-20T06:50:22,815 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085482815
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085542815
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:42629, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,816 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,818 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.108sec
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-20T06:50:22,819 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-20T06:50:22,821 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-20T06:50:22,821 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-20T06:50:22,822 INFO  [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42629,1732085421516-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-20T06:50:22,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d436e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:50:22,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,42629,-1 for getting cluster id
2024-11-20T06:50:22,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-20T06:50:22,877 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fdf0305b-36c7-467f-90c8-9a9dc96e6ba9'
2024-11-20T06:50:22,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-20T06:50:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fdf0305b-36c7-467f-90c8-9a9dc96e6ba9"

2024-11-20T06:50:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d73313f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:50:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,42629,-1]
2024-11-20T06:50:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-20T06:50:22,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:22,879 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-20T06:50:22,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a03840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-20T06:50:22,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-20T06:50:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,45189,1732085421655, seqNum=-1]
2024-11-20T06:50:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-20T06:50:22,882 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-20T06:50:22,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:22,884 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-20T06:50:22,886 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-20T06:50:22,886 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-20T06:50:22,888 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1, archiveDir=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs, maxLogs=32
2024-11-20T06:50:22,889 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732085422888
2024-11-20T06:50:22,893 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732085422888
2024-11-20T06:50:22,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:40047:40047)]
2024-11-20T06:50:22,895 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732085422894
2024-11-20T06:50:22,899 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,899 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,899 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,899 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,899 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,899 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732085422888 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732085422894
2024-11-20T06:50:22,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41363:41363),(127.0.0.1/127.0.0.1:40047:40047)]
2024-11-20T06:50:22,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732085422888 is not closed yet, will try archiving it next time
2024-11-20T06:50:22,901 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741835_1011 (size=93)
2024-11-20T06:50:22,901 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741835_1011 (size=93)
2024-11-20T06:50:22,901 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,901 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,901 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:22,901 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/WALs/test.com,8080,1/test.com%2C8080%2C1.1732085422888 to hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs/test.com%2C8080%2C1.1732085422888
2024-11-20T06:50:22,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741836_1012 (size=93)
2024-11-20T06:50:22,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741836_1012 (size=93)
2024-11-20T06:50:22,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs
2024-11-20T06:50:22,904 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732085422894)
2024-11-20T06:50:22,905 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-20T06:50:22,905 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:50:22,905 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:22,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:22,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:22,905 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-20T06:50:22,905 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-20T06:50:22,905 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1180066049, stopped=false
2024-11-20T06:50:22,905 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,42629,1732085421516
2024-11-20T06:50:22,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:22,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-20T06:50:22,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:22,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:22,910 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:50:22,911 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-20T06:50:22,911 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:22,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:22,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:22,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-20T06:50:22,911 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,45189,1732085421655' *****
2024-11-20T06:50:22,911 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-20T06:50:22,911 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-20T06:50:22,911 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-20T06:50:22,911 INFO  [RS:0;61c2066a8a0f:45189 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-20T06:50:22,911 INFO  [RS:0;61c2066a8a0f:45189 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-20T06:50:22,911 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:22,911 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:45189.
2024-11-20T06:50:22,912 DEBUG [RS:0;61c2066a8a0f:45189 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-20T06:50:22,912 DEBUG [RS:0;61c2066a8a0f:45189 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-20T06:50:22,912 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-20T06:50:22,912 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-20T06:50:22,912 DEBUG [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-20T06:50:22,912 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-20T06:50:22,912 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-20T06:50:22,912 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-20T06:50:22,912 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-20T06:50:22,912 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-20T06:50:22,912 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-20T06:50:22,928 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/.tmp/ns/671bd509777d4087857a24dd146213b1 is 43, key is default/ns:d/1732085422809/Put/seqid=0
2024-11-20T06:50:22,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741837_1013 (size=5153)
2024-11-20T06:50:22,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741837_1013 (size=5153)
2024-11-20T06:50:22,933 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/.tmp/ns/671bd509777d4087857a24dd146213b1
2024-11-20T06:50:22,937 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/.tmp/ns/671bd509777d4087857a24dd146213b1 as hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/ns/671bd509777d4087857a24dd146213b1
2024-11-20T06:50:22,941 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/ns/671bd509777d4087857a24dd146213b1, entries=2, sequenceid=6, filesize=5.0 K
2024-11-20T06:50:22,942 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false
2024-11-20T06:50:22,942 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-20T06:50:22,946 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-20T06:50:22,946 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:50:22,947 INFO  [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-20T06:50:22,947 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732085422912Running coprocessor pre-close hooks at 1732085422912Disabling compacts and flushes for region at 1732085422912Disabling writes for close at 1732085422912Obtaining lock to block concurrent updates at 1732085422912Preparing flush snapshotting stores in 1588230740 at 1732085422912Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732085422913 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085422913Flushing 1588230740/ns: creating writer at 1732085422913Flushing 1588230740/ns: appending metadata at 1732085422928 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732085422928Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73122197: reopening flushed file at 1732085422937 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1732085422942 (+5 ms)Writing region close event to WAL at 1732085422943 (+1 ms)Running coprocessor post-close hooks at 1732085422946 (+3 ms)Closed at 1732085422947 (+1 ms)
2024-11-20T06:50:22,947 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-20T06:50:23,112 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,45189,1732085421655; all regions closed.
2024-11-20T06:50:23,113 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,113 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,113 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,113 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,114 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741834_1010 (size=1152)
2024-11-20T06:50:23,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741834_1010 (size=1152)
2024-11-20T06:50:23,120 DEBUG [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs
2024-11-20T06:50:23,120 INFO  [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C45189%2C1732085421655.meta:.meta(num 1732085422724)
2024-11-20T06:50:23,121 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,121 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,121 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741833_1009 (size=93)
2024-11-20T06:50:23,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741833_1009 (size=93)
2024-11-20T06:50:23,128 DEBUG [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/oldWALs
2024-11-20T06:50:23,128 INFO  [RS:0;61c2066a8a0f:45189 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 61c2066a8a0f%2C45189%2C1732085421655:(num 1732085422304)
2024-11-20T06:50:23,128 DEBUG [RS:0;61c2066a8a0f:45189 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-20T06:50:23,128 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.LeaseManager(133): Closed leases
2024-11-20T06:50:23,128 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:50:23,129 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-20T06:50:23,129 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:50:23,129 INFO  [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:50:23,129 INFO  [RS:0;61c2066a8a0f:45189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45189
2024-11-20T06:50:23,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-20T06:50:23,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,45189,1732085421655
2024-11-20T06:50:23,152 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:50:23,160 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,45189,1732085421655]
2024-11-20T06:50:23,169 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,45189,1732085421655 already deleted, retry=false
2024-11-20T06:50:23,169 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,45189,1732085421655 expired; onlineServers=0
2024-11-20T06:50:23,169 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,42629,1732085421516' *****
2024-11-20T06:50:23,169 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-20T06:50:23,169 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-20T06:50:23,169 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-20T06:50:23,169 DEBUG [M:0;61c2066a8a0f:42629 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-20T06:50:23,170 DEBUG [M:0;61c2066a8a0f:42629 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-20T06:50:23,170 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-20T06:50:23,170 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085422057 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085422057,5,FailOnTimeoutGroup]
2024-11-20T06:50:23,170 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085422057 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085422057,5,FailOnTimeoutGroup]
2024-11-20T06:50:23,170 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-20T06:50:23,170 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-20T06:50:23,170 DEBUG [M:0;61c2066a8a0f:42629 {}] master.HMaster(1795): Stopping service threads
2024-11-20T06:50:23,170 INFO  [M:0;61c2066a8a0f:42629 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-20T06:50:23,171 INFO  [M:0;61c2066a8a0f:42629 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-20T06:50:23,171 INFO  [M:0;61c2066a8a0f:42629 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-20T06:50:23,171 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-20T06:50:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-20T06:50:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-20T06:50:23,177 DEBUG [M:0;61c2066a8a0f:42629 {}] zookeeper.ZKUtil(347): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-20T06:50:23,178 WARN  [M:0;61c2066a8a0f:42629 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-20T06:50:23,178 INFO  [M:0;61c2066a8a0f:42629 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/.lastflushedseqids
2024-11-20T06:50:23,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741838_1014 (size=99)
2024-11-20T06:50:23,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741838_1014 (size=99)
2024-11-20T06:50:23,186 INFO  [M:0;61c2066a8a0f:42629 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-20T06:50:23,186 INFO  [M:0;61c2066a8a0f:42629 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-20T06:50:23,187 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-20T06:50:23,187 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:23,187 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:23,187 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-20T06:50:23,187 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:23,187 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-20T06:50:23,208 DEBUG [M:0;61c2066a8a0f:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9647c2784c1f4d64b204abb432e140a1 is 82, key is hbase:meta,,1/info:regioninfo/1732085422749/Put/seqid=0
2024-11-20T06:50:23,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741839_1015 (size=5672)
2024-11-20T06:50:23,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741839_1015 (size=5672)
2024-11-20T06:50:23,212 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9647c2784c1f4d64b204abb432e140a1
2024-11-20T06:50:23,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-20T06:50:23,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-20T06:50:23,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-20T06:50:23,230 DEBUG [M:0;61c2066a8a0f:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a41917cf751448189efebc13c5aeb76b is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732085422813/Put/seqid=0
2024-11-20T06:50:23,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741840_1016 (size=5275)
2024-11-20T06:50:23,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741840_1016 (size=5275)
2024-11-20T06:50:23,235 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a41917cf751448189efebc13c5aeb76b
2024-11-20T06:50:23,251 DEBUG [M:0;61c2066a8a0f:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be065bf3341e495b87b50791e64aca48 is 69, key is 61c2066a8a0f,45189,1732085421655/rs:state/1732085422136/Put/seqid=0
2024-11-20T06:50:23,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741841_1017 (size=5156)
2024-11-20T06:50:23,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741841_1017 (size=5156)
2024-11-20T06:50:23,255 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be065bf3341e495b87b50791e64aca48
2024-11-20T06:50:23,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:23,261 INFO  [RS:0;61c2066a8a0f:45189 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:50:23,261 INFO  [RS:0;61c2066a8a0f:45189 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,45189,1732085421655; zookeeper connection closed.
2024-11-20T06:50:23,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45189-0x101584ba87f0001, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:23,261 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45e7ebeb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45e7ebeb
2024-11-20T06:50:23,261 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-20T06:50:23,271 DEBUG [M:0;61c2066a8a0f:42629 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c8baf6775b46481f965db10cf1ee211a is 52, key is load_balancer_on/state:d/1732085422885/Put/seqid=0
2024-11-20T06:50:23,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741842_1018 (size=5056)
2024-11-20T06:50:23,275 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741842_1018 (size=5056)
2024-11-20T06:50:23,275 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c8baf6775b46481f965db10cf1ee211a
2024-11-20T06:50:23,279 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9647c2784c1f4d64b204abb432e140a1 as hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9647c2784c1f4d64b204abb432e140a1
2024-11-20T06:50:23,283 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9647c2784c1f4d64b204abb432e140a1, entries=8, sequenceid=29, filesize=5.5 K
2024-11-20T06:50:23,284 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a41917cf751448189efebc13c5aeb76b as hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a41917cf751448189efebc13c5aeb76b
2024-11-20T06:50:23,287 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a41917cf751448189efebc13c5aeb76b, entries=3, sequenceid=29, filesize=5.2 K
2024-11-20T06:50:23,288 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be065bf3341e495b87b50791e64aca48 as hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be065bf3341e495b87b50791e64aca48
2024-11-20T06:50:23,291 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be065bf3341e495b87b50791e64aca48, entries=1, sequenceid=29, filesize=5.0 K
2024-11-20T06:50:23,292 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c8baf6775b46481f965db10cf1ee211a as hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c8baf6775b46481f965db10cf1ee211a
2024-11-20T06:50:23,296 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39117/user/jenkins/test-data/e8f6403f-8a4e-e697-8cce-1d163d32bd9f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c8baf6775b46481f965db10cf1ee211a, entries=1, sequenceid=29, filesize=4.9 K
2024-11-20T06:50:23,297 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false
2024-11-20T06:50:23,298 INFO  [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-20T06:50:23,298 DEBUG [M:0;61c2066a8a0f:42629 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732085423186Disabling compacts and flushes for region at 1732085423186Disabling writes for close at 1732085423187 (+1 ms)Obtaining lock to block concurrent updates at 1732085423187Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085423187Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732085423187Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085423188 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085423188Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085423207 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085423207Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085423216 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085423230 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085423230Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085423238 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085423250 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085423250Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732085423258 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732085423270 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732085423270Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b07eba4: reopening flushed file at 1732085423278 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bd4c72e: reopening flushed file at 1732085423283 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69231c82: reopening flushed file at 1732085423287 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f17a2cb: reopening flushed file at 1732085423292 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=29, compaction requested=false at 1732085423297 (+5 ms)Writing region close event to WAL at 1732085423298 (+1 ms)Closed at 1732085423298
2024-11-20T06:50:23,299 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,299 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,299 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,299 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,299 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-20T06:50:23,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45141 is added to blk_1073741830_1006 (size=10311)
2024-11-20T06:50:23,301 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32907 is added to blk_1073741830_1006 (size=10311)
2024-11-20T06:50:23,301 INFO  [M:0;61c2066a8a0f:42629 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-20T06:50:23,301 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-20T06:50:23,301 INFO  [M:0;61c2066a8a0f:42629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42629
2024-11-20T06:50:23,301 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-20T06:50:23,398 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:45393/user/jenkins/test-data/7d27366a-f4b3-9cd3-e476-441f8864117a/WALs/61c2066a8a0f,41993,1732085227061/61c2066a8a0f%2C41993%2C1732085227061.meta.1732085228142.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-20T06:50:23,427 INFO  [M:0;61c2066a8a0f:42629 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-20T06:50:23,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:23,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42629-0x101584ba87f0000, quorum=127.0.0.1:63817, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-20T06:50:23,430 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a9a5fda{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:23,431 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5eea892a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:23,431 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:23,431 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a1af98b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:23,431 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@392cf3b8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:23,433 WARN  [BP-1480802988-172.17.0.2-1732085419835 heartbeating to localhost/127.0.0.1:39117 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:50:23,433 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:50:23,433 WARN  [BP-1480802988-172.17.0.2-1732085419835 heartbeating to localhost/127.0.0.1:39117 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1480802988-172.17.0.2-1732085419835 (Datanode Uuid 7c8f28ff-91f6-4e8d-a143-0e077de0c49e) service to localhost/127.0.0.1:39117
2024-11-20T06:50:23,433 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:50:23,434 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data3/current/BP-1480802988-172.17.0.2-1732085419835 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:23,435 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data4/current/BP-1480802988-172.17.0.2-1732085419835 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:23,435 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:50:23,438 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12ac7c6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-20T06:50:23,439 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9268180{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:23,439 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:23,439 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9090c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:23,439 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@352a3917{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:23,440 WARN  [BP-1480802988-172.17.0.2-1732085419835 heartbeating to localhost/127.0.0.1:39117 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-20T06:50:23,440 WARN  [BP-1480802988-172.17.0.2-1732085419835 heartbeating to localhost/127.0.0.1:39117 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1480802988-172.17.0.2-1732085419835 (Datanode Uuid 14814c17-3547-4d58-9685-864590542ba5) service to localhost/127.0.0.1:39117
2024-11-20T06:50:23,440 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-20T06:50:23,441 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-20T06:50:23,441 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data1/current/BP-1480802988-172.17.0.2-1732085419835 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:23,441 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/cluster_08d259a4-29e8-8d92-74ed-e53e3bf6953b/data/data2/current/BP-1480802988-172.17.0.2-1732085419835 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-20T06:50:23,442 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-20T06:50:23,447 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7043bc8e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-20T06:50:23,448 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5eeda3c1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-20T06:50:23,448 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-20T06:50:23,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e290644{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-20T06:50:23,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@246d323a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/afba4686-6102-5cc3-e4f6-bb8908378827/hadoop.log.dir/,STOPPED}
2024-11-20T06:50:23,455 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-20T06:50:23,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-20T06:50:23,479 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=269 (was 230)
Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39117
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:39117 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:39117
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39117
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39117
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39117
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:39117 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (1210967216) connection to localhost/127.0.0.1:39117 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=536 (was 509) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=4052 (was 4351)